summaryrefslogtreecommitdiffstats
path: root/Lib
diff options
context:
space:
mode:
Diffstat (limited to 'Lib')
-rw-r--r--Lib/_collections_abc.py76
-rw-r--r--Lib/dis.py2
-rw-r--r--Lib/inspect.py26
-rw-r--r--Lib/lib2to3/Grammar.txt8
-rwxr-xr-xLib/lib2to3/pgen2/token.py6
-rw-r--r--Lib/lib2to3/pgen2/tokenize.py63
-rw-r--r--Lib/lib2to3/tests/test_parser.py32
-rw-r--r--Lib/opcode.py10
-rw-r--r--Lib/test/badsyntax_async1.py3
-rw-r--r--Lib/test/badsyntax_async2.py3
-rw-r--r--Lib/test/badsyntax_async3.py2
-rw-r--r--Lib/test/badsyntax_async4.py2
-rw-r--r--Lib/test/badsyntax_async5.py2
-rw-r--r--Lib/test/badsyntax_async6.py2
-rw-r--r--Lib/test/badsyntax_async7.py2
-rw-r--r--Lib/test/badsyntax_async8.py2
-rw-r--r--Lib/test/badsyntax_async9.py2
-rw-r--r--Lib/test/exception_hierarchy.txt1
-rw-r--r--Lib/test/test_ast.py9
-rw-r--r--Lib/test/test_collections.py80
-rw-r--r--Lib/test/test_coroutines.py862
-rw-r--r--Lib/test/test_dis.py46
-rw-r--r--Lib/test/test_grammar.py87
-rw-r--r--Lib/test/test_inspect.py72
-rw-r--r--Lib/test/test_minidom.py15
-rw-r--r--Lib/test/test_parser.py16
-rw-r--r--Lib/test/test_pickle.py4
-rw-r--r--Lib/test/test_sys.py4
-rw-r--r--Lib/test/test_tokenize.py186
-rw-r--r--Lib/test/test_types.py35
-rw-r--r--Lib/token.py6
-rw-r--r--Lib/tokenize.py56
-rw-r--r--Lib/types.py24
-rw-r--r--Lib/xml/dom/xmlbuilder.py26
34 files changed, 1734 insertions, 38 deletions
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
index cb87e6b..ea80403 100644
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -9,7 +9,8 @@ Unit tests are in test_collections.
from abc import ABCMeta, abstractmethod
import sys
-__all__ = ["Hashable", "Iterable", "Iterator", "Generator",
+__all__ = ["Awaitable", "Coroutine",
+ "Hashable", "Iterable", "Iterator", "Generator",
"Sized", "Container", "Callable",
"Set", "MutableSet",
"Mapping", "MutableMapping",
@@ -74,6 +75,79 @@ class Hashable(metaclass=ABCMeta):
return NotImplemented
+class _CoroutineMeta(ABCMeta):
+
+ def __instancecheck__(cls, instance):
+ # 0x80 = CO_COROUTINE
+ # 0x100 = CO_ITERABLE_COROUTINE
+ # We don't want to import 'inspect' module, as
+ # a dependency for 'collections.abc'.
+ CO_COROUTINES = 0x80 | 0x100
+
+ if (isinstance(instance, generator) and
+ instance.gi_code.co_flags & CO_COROUTINES):
+
+ return True
+
+ return super().__instancecheck__(instance)
+
+
+class Coroutine(metaclass=_CoroutineMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def send(self, value):
+ """Send a value into the coroutine.
+ Return next yielded value or raise StopIteration.
+ """
+ raise StopIteration
+
+ @abstractmethod
+ def throw(self, typ, val=None, tb=None):
+ """Raise an exception in the coroutine.
+ Return next yielded value or raise StopIteration.
+ """
+ if val is None:
+ if tb is None:
+ raise typ
+ val = typ()
+ if tb is not None:
+ val = val.with_traceback(tb)
+ raise val
+
+ def close(self):
+ """Raise GeneratorExit inside coroutine.
+ """
+ try:
+ self.throw(GeneratorExit)
+ except (GeneratorExit, StopIteration):
+ pass
+ else:
+ raise RuntimeError("coroutine ignored GeneratorExit")
+
+
+class Awaitable(metaclass=_CoroutineMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __await__(self):
+ yield
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Awaitable:
+ for B in C.__mro__:
+ if "__await__" in B.__dict__:
+ if B.__dict__["__await__"]:
+ return True
+ break
+ return NotImplemented
+
+Awaitable.register(Coroutine)
+
+
class Iterable(metaclass=ABCMeta):
__slots__ = ()
diff --git a/Lib/dis.py b/Lib/dis.py
index d215bc5..af37cdf 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -84,6 +84,8 @@ COMPILER_FLAG_NAMES = {
16: "NESTED",
32: "GENERATOR",
64: "NOFREE",
+ 128: "COROUTINE",
+ 256: "ITERABLE_COROUTINE",
}
def pretty_flags(flags):
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 60890f2..e52d86e 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -33,6 +33,7 @@ __author__ = ('Ka-Ping Yee <ping@lfw.org>',
import ast
import dis
+import collections.abc
import enum
import importlib.machinery
import itertools
@@ -174,7 +175,22 @@ def isgeneratorfunction(object):
See help(isfunction) for attributes listing."""
return bool((isfunction(object) or ismethod(object)) and
- object.__code__.co_flags & CO_GENERATOR)
+ object.__code__.co_flags & CO_GENERATOR and
+ not object.__code__.co_flags & CO_COROUTINE)
+
+def iscoroutinefunction(object):
+ """Return true if the object is a coroutine function.
+
+ Coroutine functions are defined with "async def" syntax,
+ or generators decorated with "types.coroutine".
+ """
+ return bool((isfunction(object) or ismethod(object)) and
+ object.__code__.co_flags & (CO_ITERABLE_COROUTINE |
+ CO_COROUTINE))
+
+def isawaitable(object):
+ """Return true if the object can be used in "await" expression."""
+ return isinstance(object, collections.abc.Awaitable)
def isgenerator(object):
"""Return true if the object is a generator.
@@ -191,7 +207,13 @@ def isgenerator(object):
send resumes the generator and "sends" a value that becomes
the result of the current yield-expression
throw used to raise an exception inside the generator"""
- return isinstance(object, types.GeneratorType)
+ return (isinstance(object, types.GeneratorType) and
+ not object.gi_code.co_flags & CO_COROUTINE)
+
+def iscoroutine(object):
+ """Return true if the object is a coroutine."""
+ return (isinstance(object, types.GeneratorType) and
+ object.gi_code.co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))
def istraceback(object):
"""Return true if the object is a traceback.
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index ce95d26..c954669 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -33,7 +33,8 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
@@ -82,7 +83,8 @@ global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -121,7 +123,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: [AWAIT] atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 7599396..1a67955 100755
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -62,8 +62,10 @@ OP = 52
COMMENT = 53
NL = 54
RARROW = 55
-ERRORTOKEN = 56
-N_TOKENS = 57
+AWAIT = 56
+ASYNC = 57
+ERRORTOKEN = 58
+N_TOKENS = 59
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 3dd1ee9..690fec4 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -220,7 +220,7 @@ class Untokenizer:
for tok in iterable:
toknum, tokval = tok[:2]
- if toknum in (NAME, NUMBER):
+ if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
if toknum == INDENT:
@@ -366,6 +366,10 @@ def generate_tokens(readline):
contline = None
indents = [0]
+ # 'stashed' and 'ctx' are used for async/await parsing
+ stashed = None
+ ctx = [('sync', 0)]
+
while 1: # loop over lines in stream
try:
line = readline()
@@ -406,6 +410,10 @@ def generate_tokens(readline):
pos = pos + 1
if pos == max: break
+ if stashed:
+ yield stashed
+ stashed = None
+
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
@@ -449,9 +457,15 @@ def generate_tokens(readline):
newline = NEWLINE
if parenlev > 0:
newline = NL
+ if stashed:
+ yield stashed
+ stashed = None
yield (newline, token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
+ if stashed:
+ yield stashed
+ stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
@@ -459,6 +473,9 @@ def generate_tokens(readline):
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
@@ -476,22 +493,64 @@ def generate_tokens(readline):
contline = line
break
else: # ordinary string
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
- yield (NAME, token, spos, epos, line)
+ if token in ('async', 'await'):
+ if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+ yield (ASYNC if token == 'async' else AWAIT,
+ token, spos, epos, line)
+ continue
+
+ tok = (NAME, token, spos, epos, line)
+ if token == 'async' and not stashed:
+ stashed = tok
+ continue
+
+ if token == 'def':
+ if (stashed
+ and stashed[0] == NAME
+ and stashed[1] == 'async'):
+
+ ctx.append(('async', indents[-1]))
+
+ yield (ASYNC, stashed[1],
+ stashed[2], stashed[3],
+ stashed[4])
+ stashed = None
+ else:
+ ctx.append(('sync', indents[-1]))
+
+ if stashed:
+ yield stashed
+ stashed = None
+
+ yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
+ if stashed:
+ yield stashed
+ stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
+ if stashed:
+ yield stashed
+ stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
+ if stashed:
+ yield stashed
+ stashed = None
+
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 5bb9d2b..107b5ab 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -55,12 +55,42 @@ class TestMatrixMultiplication(GrammarTest):
class TestYieldFrom(GrammarTest):
- def test_matrix_multiplication_operator(self):
+ def test_yield_from(self):
self.validate("yield from x")
self.validate("(yield from x) + y")
self.invalid_syntax("yield from")
+class TestAsyncAwait(GrammarTest):
+ def test_await_expr(self):
+ self.validate("""async def foo():
+ await x
+ """)
+
+ self.invalid_syntax("await x")
+ self.invalid_syntax("""def foo():
+ await x""")
+
+ def test_async_var(self):
+ self.validate("""async = 1""")
+ self.validate("""await = 1""")
+ self.validate("""def async(): pass""")
+
+ def test_async_with(self):
+ self.validate("""async def foo():
+ async for a in b: pass""")
+
+ self.invalid_syntax("""def foo():
+ async for a in b: pass""")
+
+ def test_async_for(self):
+ self.validate("""async def foo():
+ async with a: pass""")
+
+ self.invalid_syntax("""def foo():
+ async with a: pass""")
+
+
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):
self.validate("raise")
diff --git a/Lib/opcode.py b/Lib/opcode.py
index c25afd4..24e6c3f 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -85,6 +85,10 @@ def_op('BINARY_TRUE_DIVIDE', 27)
def_op('INPLACE_FLOOR_DIVIDE', 28)
def_op('INPLACE_TRUE_DIVIDE', 29)
+def_op('GET_AITER', 50)
+def_op('GET_ANEXT', 51)
+def_op('BEFORE_ASYNC_WITH', 52)
+
def_op('STORE_MAP', 54)
def_op('INPLACE_ADD', 55)
def_op('INPLACE_SUBTRACT', 56)
@@ -104,6 +108,7 @@ def_op('GET_ITER', 68)
def_op('PRINT_EXPR', 70)
def_op('LOAD_BUILD_CLASS', 71)
def_op('YIELD_FROM', 72)
+def_op('GET_AWAITABLE', 73)
def_op('INPLACE_LSHIFT', 75)
def_op('INPLACE_RSHIFT', 76)
@@ -111,7 +116,8 @@ def_op('INPLACE_AND', 77)
def_op('INPLACE_XOR', 78)
def_op('INPLACE_OR', 79)
def_op('BREAK_LOOP', 80)
-def_op('WITH_CLEANUP', 81)
+def_op('WITH_CLEANUP_START', 81)
+def_op('WITH_CLEANUP_FINISH', 82)
def_op('RETURN_VALUE', 83)
def_op('IMPORT_STAR', 84)
@@ -197,6 +203,8 @@ def_op('MAP_ADD', 147)
def_op('LOAD_CLASSDEREF', 148)
hasfree.append(148)
+jrel_op('SETUP_ASYNC_WITH', 154)
+
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
diff --git a/Lib/test/badsyntax_async1.py b/Lib/test/badsyntax_async1.py
new file mode 100644
index 0000000..970445d
--- /dev/null
+++ b/Lib/test/badsyntax_async1.py
@@ -0,0 +1,3 @@
+async def foo():
+ def foo(a=await something()):
+ pass
diff --git a/Lib/test/badsyntax_async2.py b/Lib/test/badsyntax_async2.py
new file mode 100644
index 0000000..1e62a3e
--- /dev/null
+++ b/Lib/test/badsyntax_async2.py
@@ -0,0 +1,3 @@
+async def foo():
+ def foo(a:await something()):
+ pass
diff --git a/Lib/test/badsyntax_async3.py b/Lib/test/badsyntax_async3.py
new file mode 100644
index 0000000..dde1bc5
--- /dev/null
+++ b/Lib/test/badsyntax_async3.py
@@ -0,0 +1,2 @@
+async def foo():
+ [i async for i in els]
diff --git a/Lib/test/badsyntax_async4.py b/Lib/test/badsyntax_async4.py
new file mode 100644
index 0000000..4afda40
--- /dev/null
+++ b/Lib/test/badsyntax_async4.py
@@ -0,0 +1,2 @@
+async def foo():
+ async def foo(): await something()
diff --git a/Lib/test/badsyntax_async5.py b/Lib/test/badsyntax_async5.py
new file mode 100644
index 0000000..9d19af6
--- /dev/null
+++ b/Lib/test/badsyntax_async5.py
@@ -0,0 +1,2 @@
+def foo():
+ await something()
diff --git a/Lib/test/badsyntax_async6.py b/Lib/test/badsyntax_async6.py
new file mode 100644
index 0000000..cb0a23d
--- /dev/null
+++ b/Lib/test/badsyntax_async6.py
@@ -0,0 +1,2 @@
+async def foo():
+ yield
diff --git a/Lib/test/badsyntax_async7.py b/Lib/test/badsyntax_async7.py
new file mode 100644
index 0000000..51e4bf9
--- /dev/null
+++ b/Lib/test/badsyntax_async7.py
@@ -0,0 +1,2 @@
+async def foo():
+ yield from []
diff --git a/Lib/test/badsyntax_async8.py b/Lib/test/badsyntax_async8.py
new file mode 100644
index 0000000..3c636f9
--- /dev/null
+++ b/Lib/test/badsyntax_async8.py
@@ -0,0 +1,2 @@
+async def foo():
+ await await fut
diff --git a/Lib/test/badsyntax_async9.py b/Lib/test/badsyntax_async9.py
new file mode 100644
index 0000000..d033b28
--- /dev/null
+++ b/Lib/test/badsyntax_async9.py
@@ -0,0 +1,2 @@
+async def foo():
+ await
diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt
index 1c1f69f..6632826 100644
--- a/Lib/test/exception_hierarchy.txt
+++ b/Lib/test/exception_hierarchy.txt
@@ -4,6 +4,7 @@ BaseException
+-- GeneratorExit
+-- Exception
+-- StopIteration
+ +-- StopAsyncIteration
+-- ArithmeticError
| +-- FloatingPointError
| +-- OverflowError
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 1d9de2c..c220a3c 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -106,6 +106,12 @@ exec_tests = [
"{r for l in x if g}",
# setcomp with naked tuple
"{r for l,m in x}",
+ # AsyncFunctionDef
+ "async def f():\n await something()",
+ # AsyncFor
+ "async def f():\n async for e in i: 1\n else: 2",
+ # AsyncWith
+ "async def f():\n async with a as b: 1",
]
# These are compiled through "single"
@@ -974,6 +980,9 @@ exec_results = [
('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Await', (2, 1), ('Call', (2, 7), ('Name', (2, 7), 'something', ('Load',)), [], [])))], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
]
single_results = [
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py
index 5b2e81f..ad94fdd 100644
--- a/Lib/test/test_collections.py
+++ b/Lib/test/test_collections.py
@@ -11,9 +11,11 @@ from random import randrange, shuffle
import keyword
import re
import sys
+import types
from collections import UserDict
from collections import ChainMap
from collections import deque
+from collections.abc import Awaitable, Coroutine
from collections.abc import Hashable, Iterable, Iterator, Generator
from collections.abc import Sized, Container, Callable
from collections.abc import Set, MutableSet
@@ -446,6 +448,84 @@ class ABCTestCase(unittest.TestCase):
class TestOneTrickPonyABCs(ABCTestCase):
+ def test_Awaitable(self):
+ def gen():
+ yield
+
+ @types.coroutine
+ def coro():
+ yield
+
+ async def new_coro():
+ pass
+
+ class Bar:
+ def __await__(self):
+ yield
+
+ class MinimalCoro(Coroutine):
+ def send(self, value):
+ return value
+ def throw(self, typ, val=None, tb=None):
+ super().throw(typ, val, tb)
+
+ non_samples = [None, int(), gen(), object()]
+ for x in non_samples:
+ self.assertNotIsInstance(x, Awaitable)
+ self.assertFalse(issubclass(type(x), Awaitable), repr(type(x)))
+
+ samples = [Bar(), MinimalCoro()]
+ for x in samples:
+ self.assertIsInstance(x, Awaitable)
+ self.assertTrue(issubclass(type(x), Awaitable))
+
+ c = coro()
+ self.assertIsInstance(c, Awaitable)
+ c.close() # awoid RuntimeWarning that coro() was not awaited
+
+ c = new_coro()
+ self.assertIsInstance(c, Awaitable)
+ c.close() # awoid RuntimeWarning that coro() was not awaited
+
+ def test_Coroutine(self):
+ def gen():
+ yield
+
+ @types.coroutine
+ def coro():
+ yield
+
+ async def new_coro():
+ pass
+
+ class Bar:
+ def __await__(self):
+ yield
+
+ class MinimalCoro(Coroutine):
+ def send(self, value):
+ return value
+ def throw(self, typ, val=None, tb=None):
+ super().throw(typ, val, tb)
+
+ non_samples = [None, int(), gen(), object(), Bar()]
+ for x in non_samples:
+ self.assertNotIsInstance(x, Coroutine)
+ self.assertFalse(issubclass(type(x), Coroutine), repr(type(x)))
+
+ samples = [MinimalCoro()]
+ for x in samples:
+ self.assertIsInstance(x, Awaitable)
+ self.assertTrue(issubclass(type(x), Awaitable))
+
+ c = coro()
+ self.assertIsInstance(c, Coroutine)
+ c.close() # awoid RuntimeWarning that coro() was not awaited
+
+ c = new_coro()
+ self.assertIsInstance(c, Coroutine)
+ c.close() # awoid RuntimeWarning that coro() was not awaited
+
def test_Hashable(self):
# Check some non-hashables
non_samples = [bytearray(), list(), set(), dict()]
diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py
new file mode 100644
index 0000000..2a45225
--- /dev/null
+++ b/Lib/test/test_coroutines.py
@@ -0,0 +1,862 @@
+import contextlib
+import gc
+import sys
+import types
+import unittest
+import warnings
+from test import support
+
+
+class AsyncYieldFrom:
+ def __init__(self, obj):
+ self.obj = obj
+
+ def __await__(self):
+ yield from self.obj
+
+
+class AsyncYield:
+ def __init__(self, value):
+ self.value = value
+
+ def __await__(self):
+ yield self.value
+
+
+def run_async(coro):
+ assert coro.__class__ is types.GeneratorType
+
+ buffer = []
+ result = None
+ while True:
+ try:
+ buffer.append(coro.send(None))
+ except StopIteration as ex:
+ result = ex.args[0] if ex.args else None
+ break
+ return buffer, result
+
+
+@contextlib.contextmanager
+def silence_coro_gc():
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ yield
+ support.gc_collect()
+
+
+class AsyncBadSyntaxTest(unittest.TestCase):
+
+ def test_badsyntax_1(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async1
+
+ def test_badsyntax_2(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async2
+
+ def test_badsyntax_3(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async3
+
+ def test_badsyntax_4(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async4
+
+ def test_badsyntax_5(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async5
+
+ def test_badsyntax_6(self):
+ with self.assertRaisesRegex(
+ SyntaxError, "'yield' inside async function"):
+
+ import test.badsyntax_async6
+
+ def test_badsyntax_7(self):
+ with self.assertRaisesRegex(
+ SyntaxError, "'yield from' inside async function"):
+
+ import test.badsyntax_async7
+
+ def test_badsyntax_8(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async8
+
+ def test_badsyntax_9(self):
+ with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
+ import test.badsyntax_async9
+
+
+class CoroutineTest(unittest.TestCase):
+
+ def test_gen_1(self):
+ def gen(): yield
+ self.assertFalse(hasattr(gen, '__await__'))
+
+ def test_func_1(self):
+ async def foo():
+ return 10
+
+ f = foo()
+ self.assertIsInstance(f, types.GeneratorType)
+ self.assertTrue(bool(foo.__code__.co_flags & 0x80))
+ self.assertTrue(bool(foo.__code__.co_flags & 0x20))
+ self.assertTrue(bool(f.gi_code.co_flags & 0x80))
+ self.assertTrue(bool(f.gi_code.co_flags & 0x20))
+ self.assertEqual(run_async(f), ([], 10))
+
+ def bar(): pass
+ self.assertFalse(bool(bar.__code__.co_flags & 0x80))
+
+ def test_func_2(self):
+ async def foo():
+ raise StopIteration
+
+ with self.assertRaisesRegex(
+ RuntimeError, "generator raised StopIteration"):
+
+ run_async(foo())
+
+ def test_func_3(self):
+ async def foo():
+ raise StopIteration
+
+ with silence_coro_gc():
+ self.assertRegex(repr(foo()), '^<coroutine object.* at 0x.*>$')
+
+ def test_func_4(self):
+ async def foo():
+ raise StopIteration
+
+ check = lambda: self.assertRaisesRegex(
+ TypeError, "coroutine-objects do not support iteration")
+
+ with check():
+ list(foo())
+
+ with check():
+ tuple(foo())
+
+ with check():
+ sum(foo())
+
+ with check():
+ iter(foo())
+
+ with check():
+ next(foo())
+
+ with silence_coro_gc(), check():
+ for i in foo():
+ pass
+
+ with silence_coro_gc(), check():
+ [i for i in foo()]
+
+ def test_func_5(self):
+ @types.coroutine
+ def bar():
+ yield 1
+
+ async def foo():
+ await bar()
+
+ check = lambda: self.assertRaisesRegex(
+ TypeError, "coroutine-objects do not support iteration")
+
+ with check():
+ for el in foo(): pass
+
+ # the following should pass without an error
+ for el in bar():
+ self.assertEqual(el, 1)
+ self.assertEqual([el for el in bar()], [1])
+ self.assertEqual(tuple(bar()), (1,))
+ self.assertEqual(next(iter(bar())), 1)
+
+ def test_func_6(self):
+ @types.coroutine
+ def bar():
+ yield 1
+ yield 2
+
+ async def foo():
+ await bar()
+
+ f = foo()
+ self.assertEquals(f.send(None), 1)
+ self.assertEquals(f.send(None), 2)
+ with self.assertRaises(StopIteration):
+ f.send(None)
+
+ def test_func_7(self):
+ async def bar():
+ return 10
+
+ def foo():
+ yield from bar()
+
+ with silence_coro_gc(), self.assertRaisesRegex(
+ TypeError,
+ "cannot 'yield from' a coroutine object from a generator"):
+
+ list(foo())
+
+ def test_func_8(self):
+ @types.coroutine
+ def bar():
+ return (yield from foo())
+
+ async def foo():
+ return 'spam'
+
+ self.assertEqual(run_async(bar()), ([], 'spam') )
+
+ def test_func_9(self):
+ async def foo(): pass
+
+ with self.assertWarnsRegex(
+ RuntimeWarning, "coroutine '.*test_func_9.*foo' was never awaited"):
+
+ foo()
+ support.gc_collect()
+
+ def test_await_1(self):
+
+ async def foo():
+ await 1
+ with self.assertRaisesRegex(TypeError, "object int can.t.*await"):
+ run_async(foo())
+
+ def test_await_2(self):
+ async def foo():
+ await []
+ with self.assertRaisesRegex(TypeError, "object list can.t.*await"):
+ run_async(foo())
+
+ def test_await_3(self):
+ async def foo():
+ await AsyncYieldFrom([1, 2, 3])
+
+ self.assertEqual(run_async(foo()), ([1, 2, 3], None))
+
+ def test_await_4(self):
+ async def bar():
+ return 42
+
+ async def foo():
+ return await bar()
+
+ self.assertEqual(run_async(foo()), ([], 42))
+
+ def test_await_5(self):
+ class Awaitable:
+ def __await__(self):
+ return
+
+ async def foo():
+ return (await Awaitable())
+
+ with self.assertRaisesRegex(
+ TypeError, "__await__.*returned non-iterator of type"):
+
+ run_async(foo())
+
+ def test_await_6(self):
+ class Awaitable:
+ def __await__(self):
+ return iter([52])
+
+ async def foo():
+ return (await Awaitable())
+
+ self.assertEqual(run_async(foo()), ([52], None))
+
+ def test_await_7(self):
+ class Awaitable:
+ def __await__(self):
+ yield 42
+ return 100
+
+ async def foo():
+ return (await Awaitable())
+
+ self.assertEqual(run_async(foo()), ([42], 100))
+
+ def test_await_8(self):
+ class Awaitable:
+ pass
+
+ async def foo():
+ return (await Awaitable())
+
+ with self.assertRaisesRegex(
+ TypeError, "object Awaitable can't be used in 'await' expression"):
+
+ run_async(foo())
+
+ def test_await_9(self):
+ def wrap():
+ return bar
+
+ async def bar():
+ return 42
+
+ async def foo():
+ b = bar()
+
+ db = {'b': lambda: wrap}
+
+ class DB:
+ b = wrap
+
+ return (await bar() + await wrap()() + await db['b']()()() +
+ await bar() * 1000 + await DB.b()())
+
+ async def foo2():
+ return -await bar()
+
+ self.assertEqual(run_async(foo()), ([], 42168))
+ self.assertEqual(run_async(foo2()), ([], -42))
+
+ def test_await_10(self):
+ async def baz():
+ return 42
+
+ async def bar():
+ return baz()
+
+ async def foo():
+ return await (await bar())
+
+ self.assertEqual(run_async(foo()), ([], 42))
+
+ def test_await_11(self):
+ def ident(val):
+ return val
+
+ async def bar():
+ return 'spam'
+
+ async def foo():
+ return ident(val=await bar())
+
+ async def foo2():
+ return await bar(), 'ham'
+
+ self.assertEqual(run_async(foo2()), ([], ('spam', 'ham')))
+
+ def test_await_12(self):
+ async def coro():
+ return 'spam'
+
+ class Awaitable:
+ def __await__(self):
+ return coro()
+
+ async def foo():
+ return await Awaitable()
+
+ with self.assertRaisesRegex(
+ TypeError, "__await__\(\) returned a coroutine"):
+
+ run_async(foo())
+
+ def test_await_13(self):
+ class Awaitable:
+ def __await__(self):
+ return self
+
+ async def foo():
+ return await Awaitable()
+
+ with self.assertRaisesRegex(
+ TypeError, "__await__.*returned non-iterator of type"):
+
+ run_async(foo())
+
+ def test_with_1(self):
+ class Manager:
+ def __init__(self, name):
+ self.name = name
+
+ async def __aenter__(self):
+ await AsyncYieldFrom(['enter-1-' + self.name,
+ 'enter-2-' + self.name])
+ return self
+
+ async def __aexit__(self, *args):
+ await AsyncYieldFrom(['exit-1-' + self.name,
+ 'exit-2-' + self.name])
+
+ if self.name == 'B':
+ return True
+
+
+ async def foo():
+ async with Manager("A") as a, Manager("B") as b:
+ await AsyncYieldFrom([('managers', a.name, b.name)])
+ 1/0
+
+ f = foo()
+ result, _ = run_async(f)
+
+ self.assertEqual(
+ result, ['enter-1-A', 'enter-2-A', 'enter-1-B', 'enter-2-B',
+ ('managers', 'A', 'B'),
+ 'exit-1-B', 'exit-2-B', 'exit-1-A', 'exit-2-A']
+ )
+
+ async def foo():
+ async with Manager("A") as a, Manager("C") as c:
+ await AsyncYieldFrom([('managers', a.name, c.name)])
+ 1/0
+
+ with self.assertRaises(ZeroDivisionError):
+ run_async(foo())
+
+ def test_with_2(self):
+ class CM:
+ def __aenter__(self):
+ pass
+
+ async def foo():
+ async with CM():
+ pass
+
+ with self.assertRaisesRegex(AttributeError, '__aexit__'):
+ run_async(foo())
+
+ def test_with_3(self):
+ class CM:
+ def __aexit__(self):
+ pass
+
+ async def foo():
+ async with CM():
+ pass
+
+ with self.assertRaisesRegex(AttributeError, '__aenter__'):
+ run_async(foo())
+
+ def test_with_4(self):
+ class CM:
+ def __enter__(self):
+ pass
+
+ def __exit__(self):
+ pass
+
+ async def foo():
+ async with CM():
+ pass
+
+ with self.assertRaisesRegex(AttributeError, '__aexit__'):
+ run_async(foo())
+
+ def test_with_5(self):
+ # While this test doesn't make a lot of sense,
+ # it's a regression test for an early bug with opcodes
+ # generation
+
+ class CM:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ pass
+
+ async def func():
+ async with CM():
+ assert (1, ) == 1
+
+ with self.assertRaises(AssertionError):
+ run_async(func())
+
+ def test_with_6(self):
+ class CM:
+ def __aenter__(self):
+ return 123
+
+ def __aexit__(self, *e):
+ return 456
+
+ async def foo():
+ async with CM():
+ pass
+
+ with self.assertRaisesRegex(
+ TypeError, "object int can't be used in 'await' expression"):
+ # it's important that __aexit__ wasn't called
+ run_async(foo())
+
+ def test_with_7(self):
+ class CM:
+ async def __aenter__(self):
+ return self
+
+ def __aexit__(self, *e):
+ return 456
+
+ async def foo():
+ async with CM():
+ pass
+
+ with self.assertRaisesRegex(
+ TypeError, "object int can't be used in 'await' expression"):
+
+ run_async(foo())
+
+ def test_for_1(self):
+ aiter_calls = 0
+
+ class AsyncIter:
+ def __init__(self):
+ self.i = 0
+
+ async def __aiter__(self):
+ nonlocal aiter_calls
+ aiter_calls += 1
+ return self
+
+ async def __anext__(self):
+ self.i += 1
+
+ if not (self.i % 10):
+ await AsyncYield(self.i * 10)
+
+ if self.i > 100:
+ raise StopAsyncIteration
+
+ return self.i, self.i
+
+
+ buffer = []
+ async def test1():
+ async for i1, i2 in AsyncIter():
+ buffer.append(i1 + i2)
+
+ yielded, _ = run_async(test1())
+ # Make sure that __aiter__ was called only once
+ self.assertEqual(aiter_calls, 1)
+ self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
+ self.assertEqual(buffer, [i*2 for i in range(1, 101)])
+
+
+ buffer = []
+ async def test2():
+ nonlocal buffer
+ async for i in AsyncIter():
+ buffer.append(i[0])
+ if i[0] == 20:
+ break
+ else:
+ buffer.append('what?')
+ buffer.append('end')
+
+ yielded, _ = run_async(test2())
+ # Make sure that __aiter__ was called only once
+ self.assertEqual(aiter_calls, 2)
+ self.assertEqual(yielded, [100, 200])
+ self.assertEqual(buffer, [i for i in range(1, 21)] + ['end'])
+
+
+ buffer = []
+ async def test3():
+ nonlocal buffer
+ async for i in AsyncIter():
+ if i[0] > 20:
+ continue
+ buffer.append(i[0])
+ else:
+ buffer.append('what?')
+ buffer.append('end')
+
+ yielded, _ = run_async(test3())
+ # Make sure that __aiter__ was called only once
+ self.assertEqual(aiter_calls, 3)
+ self.assertEqual(yielded, [i * 100 for i in range(1, 11)])
+ self.assertEqual(buffer, [i for i in range(1, 21)] +
+ ['what?', 'end'])
+
+ def test_for_2(self):
+ tup = (1, 2, 3)
+ refs_before = sys.getrefcount(tup)
+
+ async def foo():
+ async for i in tup:
+ print('never going to happen')
+
+ with self.assertRaisesRegex(
+ TypeError, "async for' requires an object.*__aiter__.*tuple"):
+
+ run_async(foo())
+
+ self.assertEqual(sys.getrefcount(tup), refs_before)
+
+ def test_for_3(self):
+ class I:
+ def __aiter__(self):
+ return self
+
+ aiter = I()
+ refs_before = sys.getrefcount(aiter)
+
+ async def foo():
+ async for i in aiter:
+ print('never going to happen')
+
+ with self.assertRaisesRegex(
+ TypeError,
+ "async for' received an invalid object.*__aiter.*\: I"):
+
+ run_async(foo())
+
+ self.assertEqual(sys.getrefcount(aiter), refs_before)
+
+ def test_for_4(self):
+ class I:
+ async def __aiter__(self):
+ return self
+
+ def __anext__(self):
+ return ()
+
+ aiter = I()
+ refs_before = sys.getrefcount(aiter)
+
+ async def foo():
+ async for i in aiter:
+ print('never going to happen')
+
+ with self.assertRaisesRegex(
+ TypeError,
+ "async for' received an invalid object.*__anext__.*tuple"):
+
+ run_async(foo())
+
+ self.assertEqual(sys.getrefcount(aiter), refs_before)
+
+ def test_for_5(self):
+ class I:
+ async def __aiter__(self):
+ return self
+
+ def __anext__(self):
+ return 123
+
+ async def foo():
+ async for i in I():
+ print('never going to happen')
+
+ with self.assertRaisesRegex(
+ TypeError,
+ "async for' received an invalid object.*__anext.*int"):
+
+ run_async(foo())
+
+ def test_for_6(self):
+ I = 0
+
+ class Manager:
+ async def __aenter__(self):
+ nonlocal I
+ I += 10000
+
+ async def __aexit__(self, *args):
+ nonlocal I
+ I += 100000
+
+ class Iterable:
+ def __init__(self):
+ self.i = 0
+
+ async def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if self.i > 10:
+ raise StopAsyncIteration
+ self.i += 1
+ return self.i
+
+ ##############
+
+ manager = Manager()
+ iterable = Iterable()
+ mrefs_before = sys.getrefcount(manager)
+ irefs_before = sys.getrefcount(iterable)
+
+ async def main():
+ nonlocal I
+
+ async with manager:
+ async for i in iterable:
+ I += 1
+ I += 1000
+
+ run_async(main())
+ self.assertEqual(I, 111011)
+
+ self.assertEqual(sys.getrefcount(manager), mrefs_before)
+ self.assertEqual(sys.getrefcount(iterable), irefs_before)
+
+ ##############
+
+ async def main():
+ nonlocal I
+
+ async with Manager():
+ async for i in Iterable():
+ I += 1
+ I += 1000
+
+ async with Manager():
+ async for i in Iterable():
+ I += 1
+ I += 1000
+
+ run_async(main())
+ self.assertEqual(I, 333033)
+
+ ##############
+
+ async def main():
+ nonlocal I
+
+ async with Manager():
+ I += 100
+ async for i in Iterable():
+ I += 1
+ else:
+ I += 10000000
+ I += 1000
+
+ async with Manager():
+ I += 100
+ async for i in Iterable():
+ I += 1
+ else:
+ I += 10000000
+ I += 1000
+
+ run_async(main())
+ self.assertEqual(I, 20555255)
+
+
+class CoroAsyncIOCompatTest(unittest.TestCase):
+
+ def test_asyncio_1(self):
+ import asyncio
+
+ class MyException(Exception):
+ pass
+
+ buffer = []
+
+ class CM:
+ async def __aenter__(self):
+ buffer.append(1)
+ await asyncio.sleep(0.01)
+ buffer.append(2)
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ await asyncio.sleep(0.01)
+ buffer.append(exc_type.__name__)
+
+ async def f():
+ async with CM() as c:
+ await asyncio.sleep(0.01)
+ raise MyException
+ buffer.append('unreachable')
+
+ loop = asyncio.get_event_loop()
+ try:
+ loop.run_until_complete(f())
+ except MyException:
+ pass
+ finally:
+ loop.close()
+
+ self.assertEqual(buffer, [1, 2, 'MyException'])
+
+
+class SysSetCoroWrapperTest(unittest.TestCase):
+
+ def test_set_wrapper_1(self):
+ async def foo():
+ return 'spam'
+
+ wrapped = None
+ def wrap(gen):
+ nonlocal wrapped
+ wrapped = gen
+ return gen
+
+ self.assertIsNone(sys.get_coroutine_wrapper())
+
+ sys.set_coroutine_wrapper(wrap)
+ self.assertIs(sys.get_coroutine_wrapper(), wrap)
+ try:
+ f = foo()
+ self.assertTrue(wrapped)
+
+ self.assertEqual(run_async(f), ([], 'spam'))
+ finally:
+ sys.set_coroutine_wrapper(None)
+
+ self.assertIsNone(sys.get_coroutine_wrapper())
+
+ wrapped = None
+ with silence_coro_gc():
+ foo()
+ self.assertFalse(wrapped)
+
+ def test_set_wrapper_2(self):
+ self.assertIsNone(sys.get_coroutine_wrapper())
+ with self.assertRaisesRegex(TypeError, "callable expected, got int"):
+ sys.set_coroutine_wrapper(1)
+ self.assertIsNone(sys.get_coroutine_wrapper())
+
+
+class CAPITest(unittest.TestCase):
+
+ def test_tp_await_1(self):
+ from _testcapi import awaitType as at
+
+ async def foo():
+ future = at(iter([1]))
+ return (await future)
+
+ self.assertEqual(foo().send(None), 1)
+
+ def test_tp_await_2(self):
+ # Test tp_await to __await__ mapping
+ from _testcapi import awaitType as at
+ future = at(iter([1]))
+ self.assertEqual(next(future.__await__()), 1)
+
+ def test_tp_await_3(self):
+ from _testcapi import awaitType as at
+
+ async def foo():
+ future = at(1)
+ return (await future)
+
+ with self.assertRaisesRegex(
+ TypeError, "__await__.*returned non-iterator of type 'int'"):
+ self.assertEqual(foo().send(None), 1)
+
+
+def test_main():
+ support.run_unittest(AsyncBadSyntaxTest,
+ CoroutineTest,
+ CoroAsyncIOCompatTest,
+ SysSetCoroWrapperTest,
+ CAPITest)
+
+
+if __name__=="__main__":
+ test_main()
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 7890b1f..421bbad 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -480,6 +480,24 @@ Constants:
Names:
0: x"""
+
+async def async_def():
+ await 1
+ async for a in b: pass
+ async with c as d: pass
+
+code_info_async_def = """\
+Name: async_def
+Filename: (.*)
+Argument count: 0
+Kw-only arguments: 0
+Number of locals: 2
+Stack size: 17
+Flags: OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
+Constants:
+ 0: None
+ 1: 1"""
+
class CodeInfoTests(unittest.TestCase):
test_pairs = [
(dis.code_info, code_info_code_info),
@@ -488,6 +506,7 @@ class CodeInfoTests(unittest.TestCase):
(expr_str, code_info_expr_str),
(simple_stmt_str, code_info_simple_stmt_str),
(compound_stmt_str, code_info_compound_stmt_str),
+ (async_def, code_info_async_def)
]
def test_code_info(self):
@@ -697,7 +716,7 @@ expected_opinfo_jumpy = [
Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=135, starts_line=None, is_jump_target=False),
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=138, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=141, starts_line=None, is_jump_target=False),
- Instruction(opname='SETUP_FINALLY', opcode=122, arg=72, argval=217, argrepr='to 217', offset=142, starts_line=20, is_jump_target=True),
+ Instruction(opname='SETUP_FINALLY', opcode=122, arg=73, argval=218, argrepr='to 218', offset=142, starts_line=20, is_jump_target=True),
Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=160, argrepr='to 160', offset=145, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=148, starts_line=21, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=151, starts_line=None, is_jump_target=False),
@@ -717,7 +736,7 @@ expected_opinfo_jumpy = [
Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=179, starts_line=None, is_jump_target=False),
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
- Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=213, argrepr='to 213', offset=184, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_FORWARD', opcode=110, arg=27, argval=214, argrepr='to 214', offset=184, starts_line=None, is_jump_target=False),
Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=187, starts_line=None, is_jump_target=True),
Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=188, starts_line=25, is_jump_target=True),
Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=211, argrepr='to 211', offset=191, starts_line=None, is_jump_target=False),
@@ -728,17 +747,18 @@ expected_opinfo_jumpy = [
Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False),
Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=207, starts_line=None, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=208, starts_line=None, is_jump_target=False),
- Instruction(opname='WITH_CLEANUP', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
- Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
- Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=True),
- Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=214, starts_line=None, is_jump_target=False),
- Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=217, starts_line=28, is_jump_target=True),
- Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=220, starts_line=None, is_jump_target=False),
- Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=223, starts_line=None, is_jump_target=False),
- Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=226, starts_line=None, is_jump_target=False),
- Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
- Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=228, starts_line=None, is_jump_target=False),
- Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=231, starts_line=None, is_jump_target=False),
+ Instruction(opname='WITH_CLEANUP_START', opcode=81, arg=None, argval=None, argrepr='', offset=211, starts_line=None, is_jump_target=True),
+ Instruction(opname='WITH_CLEANUP_FINISH', opcode=82, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False),
+ Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=213, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=214, starts_line=None, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=215, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=218, starts_line=28, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=221, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=224, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=227, starts_line=None, is_jump_target=False),
+ Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=228, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=229, starts_line=None, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=232, starts_line=None, is_jump_target=False),
]
# One last piece of inspect fodder to check the default line number handling
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 28b1f04..e46a232 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -2,6 +2,7 @@
# This just tests whether the parser accepts them all.
from test.support import check_syntax_error
+import inspect
import unittest
import sys
# testing import *
@@ -1034,6 +1035,92 @@ class GrammarTests(unittest.TestCase):
m @= 42
self.assertEqual(m.other, 42)
+ def test_async_await(self):
+ async = 1
+ await = 2
+ self.assertEqual(async, 1)
+
+ def async():
+ nonlocal await
+ await = 10
+ async()
+ self.assertEqual(await, 10)
+
+ self.assertFalse(bool(async.__code__.co_flags & inspect.CO_COROUTINE))
+
+ async def test():
+ def sum():
+ async = 1
+ await = 41
+ return async + await
+
+ if 1:
+ await someobj()
+
+ self.assertEqual(test.__name__, 'test')
+ self.assertTrue(bool(test.__code__.co_flags & inspect.CO_COROUTINE))
+
+ def decorator(func):
+ setattr(func, '_marked', True)
+ return func
+
+ @decorator
+ async def test2():
+ return 22
+ self.assertTrue(test2._marked)
+ self.assertEqual(test2.__name__, 'test2')
+ self.assertTrue(bool(test2.__code__.co_flags & inspect.CO_COROUTINE))
+
+ def test_async_for(self):
+ class Done(Exception): pass
+
+ class AIter:
+ async def __aiter__(self):
+ return self
+ async def __anext__(self):
+ raise StopAsyncIteration
+
+ async def foo():
+ async for i in AIter():
+ pass
+ async for i, j in AIter():
+ pass
+ async for i in AIter():
+ pass
+ else:
+ pass
+ raise Done
+
+ with self.assertRaises(Done):
+ foo().send(None)
+
+ def test_async_with(self):
+ class Done(Exception): pass
+
+ class manager:
+ async def __aenter__(self):
+ return (1, 2)
+ async def __aexit__(self, *exc):
+ return False
+
+ async def foo():
+ async with manager():
+ pass
+ async with manager() as x:
+ pass
+ async with manager() as (x, y):
+ pass
+ async with manager(), manager():
+ pass
+ async with manager() as x, manager() as y:
+ pass
+ async with manager() as x, manager():
+ pass
+ raise Done
+
+ with self.assertRaises(Done):
+ foo().send(None)
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 43ef755..20df755 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -18,6 +18,7 @@ import textwrap
import unicodedata
import unittest
import unittest.mock
+import warnings
try:
from concurrent.futures import ThreadPoolExecutor
@@ -62,14 +63,16 @@ class IsTestBase(unittest.TestCase):
predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode,
inspect.isframe, inspect.isfunction, inspect.ismethod,
inspect.ismodule, inspect.istraceback,
- inspect.isgenerator, inspect.isgeneratorfunction])
+ inspect.isgenerator, inspect.isgeneratorfunction,
+ inspect.iscoroutine, inspect.iscoroutinefunction])
def istest(self, predicate, exp):
obj = eval(exp)
self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp))
for other in self.predicates - set([predicate]):
- if predicate == inspect.isgeneratorfunction and\
+ if (predicate == inspect.isgeneratorfunction or \
+ predicate == inspect.iscoroutinefunction) and \
other == inspect.isfunction:
continue
self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp))
@@ -78,13 +81,21 @@ def generator_function_example(self):
for i in range(2):
yield i
+async def coroutine_function_example(self):
+ return 'spam'
+
+@types.coroutine
+def gen_coroutine_function_example(self):
+ yield
+ return 'spam'
+
class TestPredicates(IsTestBase):
- def test_sixteen(self):
+ def test_eightteen(self):
count = len([x for x in dir(inspect) if x.startswith('is')])
# This test is here for remember you to update Doc/library/inspect.rst
# which claims there are 16 such functions
- expected = 16
+ expected = 19
err_msg = "There are %d (not %d) is* functions" % (count, expected)
self.assertEqual(count, expected, err_msg)
@@ -115,11 +126,64 @@ class TestPredicates(IsTestBase):
self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
self.istest(inspect.isgenerator, '(x for x in range(2))')
self.istest(inspect.isgeneratorfunction, 'generator_function_example')
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ self.istest(inspect.iscoroutine, 'coroutine_function_example(1)')
+ self.istest(inspect.iscoroutinefunction, 'coroutine_function_example')
+
if hasattr(types, 'MemberDescriptorType'):
self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
else:
self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
+ def test_iscoroutine(self):
+ gen_coro = gen_coroutine_function_example(1)
+ coro = coroutine_function_example(1)
+
+ self.assertTrue(
+ inspect.iscoroutinefunction(gen_coroutine_function_example))
+ self.assertTrue(inspect.iscoroutine(gen_coro))
+
+ self.assertTrue(
+ inspect.isgeneratorfunction(gen_coroutine_function_example))
+ self.assertTrue(inspect.isgenerator(gen_coro))
+
+ self.assertTrue(
+ inspect.iscoroutinefunction(coroutine_function_example))
+ self.assertTrue(inspect.iscoroutine(coro))
+
+ self.assertFalse(
+ inspect.isgeneratorfunction(coroutine_function_example))
+ self.assertFalse(inspect.isgenerator(coro))
+
+ coro.close(); gen_coro.close() # silence warnings
+
+ def test_isawaitable(self):
+ def gen(): yield
+ self.assertFalse(inspect.isawaitable(gen()))
+
+ coro = coroutine_function_example(1)
+ gen_coro = gen_coroutine_function_example(1)
+
+ self.assertTrue(
+ inspect.isawaitable(coro))
+ self.assertTrue(
+ inspect.isawaitable(gen_coro))
+
+ class Future:
+ def __await__():
+ pass
+ self.assertTrue(inspect.isawaitable(Future()))
+ self.assertFalse(inspect.isawaitable(Future))
+
+ class NotFuture: pass
+ not_fut = NotFuture()
+ not_fut.__await__ = lambda: None
+ self.assertFalse(inspect.isawaitable(not_fut))
+
+ coro.close(); gen_coro.close() # silence warnings
+
def test_isroutine(self):
self.assertTrue(inspect.isroutine(mod.spam))
self.assertTrue(inspect.isroutine([].count))
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
index 686638c..e8ca497 100644
--- a/Lib/test/test_minidom.py
+++ b/Lib/test/test_minidom.py
@@ -49,6 +49,21 @@ class MinidomTest(unittest.TestCase):
t = node.wholeText
self.confirm(t == s, "looking for %r, found %r" % (s, t))
+ def testDocumentAsyncAttr(self):
+ doc = Document()
+ self.assertFalse(doc.async_)
+ with self.assertWarns(DeprecationWarning):
+ self.assertFalse(getattr(doc, 'async', True))
+ with self.assertWarns(DeprecationWarning):
+ setattr(doc, 'async', True)
+ with self.assertWarns(DeprecationWarning):
+ self.assertTrue(getattr(doc, 'async', False))
+ self.assertTrue(doc.async_)
+
+ self.assertFalse(Document.async_)
+ with self.assertWarns(DeprecationWarning):
+ self.assertFalse(getattr(Document, 'async', True))
+
def testParseFromBinaryFile(self):
with open(tstfile, 'rb') as file:
dom = parse(file)
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 56b5d95..7082273 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -63,6 +63,22 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
" if (yield):\n"
" yield x\n")
+ def test_await_statement(self):
+ self.check_suite("async def f():\n await smth()")
+ self.check_suite("async def f():\n foo = await smth()")
+ self.check_suite("async def f():\n foo, bar = await smth()")
+ self.check_suite("async def f():\n (await smth())")
+ self.check_suite("async def f():\n foo((await smth()))")
+ self.check_suite("async def f():\n await foo(); return 42")
+
+ def test_async_with_statement(self):
+ self.check_suite("async def f():\n async with 1: pass")
+ self.check_suite("async def f():\n async with a as b, c as d: pass")
+
+ def test_async_for_statement(self):
+ self.check_suite("async def f():\n async for i in (): pass")
+ self.check_suite("async def f():\n async for i, b in (): pass")
+
def test_nonlocal_statement(self):
self.check_suite("def f():\n"
" x = 0\n"
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index f5add27..a9853c1 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -351,7 +351,9 @@ class CompatPickleTests(unittest.TestCase):
for name, exc in get_exceptions(builtins):
with self.subTest(name):
- if exc in (BlockingIOError, ResourceWarning):
+ if exc in (BlockingIOError,
+ ResourceWarning,
+ StopAsyncIteration):
continue
if exc is not OSError and issubclass(exc, OSError):
self.assertEqual(reverse_mapping('builtins', name),
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 6d2763b..494a53b 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1025,9 +1025,9 @@ class SizeofTest(unittest.TestCase):
# static type: PyTypeObject
s = vsize('P2n15Pl4Pn9Pn11PIP')
check(int, s)
- # (PyTypeObject + PyNumberMethods + PyMappingMethods +
+ # (PyTypeObject + PyAsyncMethods + PyNumberMethods + PyMappingMethods +
# PySequenceMethods + PyBufferProcs + 4P)
- s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 10P 2P 4P')
+ s = vsize('P2n17Pl4Pn9Pn11PIP') + struct.calcsize('34P 3P 3P 10P 2P 4P')
# Separate block for PyDictKeysObject with 4 entries
s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
# class
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 03f6148..ed75171 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -641,6 +641,192 @@ Legacy unicode literals:
NAME 'grĂ¼n' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
+
+Async/await extension:
+
+ >>> dump_tokens("async = 1")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ OP '=' (1, 6) (1, 7)
+ NUMBER '1' (1, 8) (1, 9)
+
+ >>> dump_tokens("a = (async = 1)")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'a' (1, 0) (1, 1)
+ OP '=' (1, 2) (1, 3)
+ OP '(' (1, 4) (1, 5)
+ NAME 'async' (1, 5) (1, 10)
+ OP '=' (1, 11) (1, 12)
+ NUMBER '1' (1, 13) (1, 14)
+ OP ')' (1, 14) (1, 15)
+
+ >>> dump_tokens("async()")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ OP '(' (1, 5) (1, 6)
+ OP ')' (1, 6) (1, 7)
+
+ >>> dump_tokens("class async(Bar):pass")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'class' (1, 0) (1, 5)
+ NAME 'async' (1, 6) (1, 11)
+ OP '(' (1, 11) (1, 12)
+ NAME 'Bar' (1, 12) (1, 15)
+ OP ')' (1, 15) (1, 16)
+ OP ':' (1, 16) (1, 17)
+ NAME 'pass' (1, 17) (1, 21)
+
+ >>> dump_tokens("class async:pass")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'class' (1, 0) (1, 5)
+ NAME 'async' (1, 6) (1, 11)
+ OP ':' (1, 11) (1, 12)
+ NAME 'pass' (1, 12) (1, 16)
+
+ >>> dump_tokens("await = 1")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'await' (1, 0) (1, 5)
+ OP '=' (1, 6) (1, 7)
+ NUMBER '1' (1, 8) (1, 9)
+
+ >>> dump_tokens("foo.async")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'foo' (1, 0) (1, 3)
+ OP '.' (1, 3) (1, 4)
+ NAME 'async' (1, 4) (1, 9)
+
+ >>> dump_tokens("async for a in b: pass")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ NAME 'for' (1, 6) (1, 9)
+ NAME 'a' (1, 10) (1, 11)
+ NAME 'in' (1, 12) (1, 14)
+ NAME 'b' (1, 15) (1, 16)
+ OP ':' (1, 16) (1, 17)
+ NAME 'pass' (1, 18) (1, 22)
+
+ >>> dump_tokens("async with a as b: pass")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ NAME 'with' (1, 6) (1, 10)
+ NAME 'a' (1, 11) (1, 12)
+ NAME 'as' (1, 13) (1, 15)
+ NAME 'b' (1, 16) (1, 17)
+ OP ':' (1, 17) (1, 18)
+ NAME 'pass' (1, 19) (1, 23)
+
+ >>> dump_tokens("async.foo")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ OP '.' (1, 5) (1, 6)
+ NAME 'foo' (1, 6) (1, 9)
+
+ >>> dump_tokens("async")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+
+ >>> dump_tokens("async\\n#comment\\nawait")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ NEWLINE '\\n' (1, 5) (1, 6)
+ COMMENT '#comment' (2, 0) (2, 8)
+ NL '\\n' (2, 8) (2, 9)
+ NAME 'await' (3, 0) (3, 5)
+
+ >>> dump_tokens("async\\n...\\nawait")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ NEWLINE '\\n' (1, 5) (1, 6)
+ OP '...' (2, 0) (2, 3)
+ NEWLINE '\\n' (2, 3) (2, 4)
+ NAME 'await' (3, 0) (3, 5)
+
+ >>> dump_tokens("async\\nawait")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'async' (1, 0) (1, 5)
+ NEWLINE '\\n' (1, 5) (1, 6)
+ NAME 'await' (2, 0) (2, 5)
+
+ >>> dump_tokens("foo.async + 1")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'foo' (1, 0) (1, 3)
+ OP '.' (1, 3) (1, 4)
+ NAME 'async' (1, 4) (1, 9)
+ OP '+' (1, 10) (1, 11)
+ NUMBER '1' (1, 12) (1, 13)
+
+ >>> dump_tokens("async def foo(): pass")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ ASYNC 'async' (1, 0) (1, 5)
+ NAME 'def' (1, 6) (1, 9)
+ NAME 'foo' (1, 10) (1, 13)
+ OP '(' (1, 13) (1, 14)
+ OP ')' (1, 14) (1, 15)
+ OP ':' (1, 15) (1, 16)
+ NAME 'pass' (1, 17) (1, 21)
+
+ >>> dump_tokens('''async def foo():
+ ... def foo(await):
+ ... await = 1
+ ... if 1:
+ ... await
+ ... async += 1
+ ... ''')
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ ASYNC 'async' (1, 0) (1, 5)
+ NAME 'def' (1, 6) (1, 9)
+ NAME 'foo' (1, 10) (1, 13)
+ OP '(' (1, 13) (1, 14)
+ OP ')' (1, 14) (1, 15)
+ OP ':' (1, 15) (1, 16)
+ NEWLINE '\\n' (1, 16) (1, 17)
+ INDENT ' ' (2, 0) (2, 2)
+ NAME 'def' (2, 2) (2, 5)
+ NAME 'foo' (2, 6) (2, 9)
+ OP '(' (2, 9) (2, 10)
+ NAME 'await' (2, 10) (2, 15)
+ OP ')' (2, 15) (2, 16)
+ OP ':' (2, 16) (2, 17)
+ NEWLINE '\\n' (2, 17) (2, 18)
+ INDENT ' ' (3, 0) (3, 4)
+ NAME 'await' (3, 4) (3, 9)
+ OP '=' (3, 10) (3, 11)
+ NUMBER '1' (3, 12) (3, 13)
+ NEWLINE '\\n' (3, 13) (3, 14)
+ DEDENT '' (4, 2) (4, 2)
+ NAME 'if' (4, 2) (4, 4)
+ NUMBER '1' (4, 5) (4, 6)
+ OP ':' (4, 6) (4, 7)
+ NEWLINE '\\n' (4, 7) (4, 8)
+ INDENT ' ' (5, 0) (5, 4)
+ AWAIT 'await' (5, 4) (5, 9)
+ NEWLINE '\\n' (5, 9) (5, 10)
+ DEDENT '' (6, 0) (6, 0)
+ DEDENT '' (6, 0) (6, 0)
+ NAME 'async' (6, 0) (6, 5)
+ OP '+=' (6, 6) (6, 8)
+ NUMBER '1' (6, 9) (6, 10)
+ NEWLINE '\\n' (6, 10) (6, 11)
+
+ >>> dump_tokens('''async def foo():
+ ... async for i in 1: pass''')
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ ASYNC 'async' (1, 0) (1, 5)
+ NAME 'def' (1, 6) (1, 9)
+ NAME 'foo' (1, 10) (1, 13)
+ OP '(' (1, 13) (1, 14)
+ OP ')' (1, 14) (1, 15)
+ OP ':' (1, 15) (1, 16)
+ NEWLINE '\\n' (1, 16) (1, 17)
+ INDENT ' ' (2, 0) (2, 2)
+ ASYNC 'async' (2, 2) (2, 7)
+ NAME 'for' (2, 8) (2, 11)
+ NAME 'i' (2, 12) (2, 13)
+ NAME 'in' (2, 14) (2, 16)
+ NUMBER '1' (2, 17) (2, 18)
+ OP ':' (2, 18) (2, 19)
+ NAME 'pass' (2, 20) (2, 24)
+ DEDENT '' (3, 0) (3, 0)
"""
from test import support
diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index 8cdecb0..c5a35f9 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -1,7 +1,8 @@
# Python test set -- part 6, built-in types
from test.support import run_with_locale
-import collections
+import collections.abc
+import inspect
import pickle
import locale
import sys
@@ -1172,5 +1173,37 @@ class SimpleNamespaceTests(unittest.TestCase):
self.assertEqual(ns, ns_roundtrip, pname)
+class CoroutineTests(unittest.TestCase):
+ def test_wrong_args(self):
+ class Foo:
+ def __call__(self):
+ pass
+ def bar(): pass
+
+ samples = [Foo, Foo(), bar, None, int, 1]
+ for sample in samples:
+ with self.assertRaisesRegex(TypeError, 'expects a generator'):
+ types.coroutine(sample)
+
+ def test_genfunc(self):
+ def gen():
+ yield
+
+ self.assertFalse(isinstance(gen(), collections.abc.Coroutine))
+ self.assertFalse(isinstance(gen(), collections.abc.Awaitable))
+
+ self.assertIs(types.coroutine(gen), gen)
+
+ self.assertTrue(gen.__code__.co_flags & inspect.CO_ITERABLE_COROUTINE)
+ self.assertFalse(gen.__code__.co_flags & inspect.CO_COROUTINE)
+
+ g = gen()
+ self.assertTrue(g.gi_code.co_flags & inspect.CO_ITERABLE_COROUTINE)
+ self.assertFalse(g.gi_code.co_flags & inspect.CO_COROUTINE)
+ self.assertTrue(isinstance(g, collections.abc.Coroutine))
+ self.assertTrue(isinstance(g, collections.abc.Awaitable))
+ g.close() # silence warning
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/token.py b/Lib/token.py
index a95d9b7..5fdb222 100644
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -64,8 +64,10 @@ ATEQUAL = 50
RARROW = 51
ELLIPSIS = 52
OP = 53
-ERRORTOKEN = 54
-N_TOKENS = 55
+AWAIT = 54
+ASYNC = 55
+ERRORTOKEN = 56
+N_TOKENS = 57
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 8bc83fd..d65325e 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -274,7 +274,7 @@ class Untokenizer:
self.encoding = tokval
continue
- if toknum in (NAME, NUMBER):
+ if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
# Insert a space between two consecutive strings
@@ -477,6 +477,10 @@ def _tokenize(readline, encoding):
contline = None
indents = [0]
+ # 'stashed' and 'ctx' are used for async/await parsing
+ stashed = None
+ ctx = [('sync', 0)]
+
if encoding is not None:
if encoding == "utf-8-sig":
# BOM will already have been stripped.
@@ -552,6 +556,11 @@ def _tokenize(readline, encoding):
"unindent does not match any outer indentation level",
("<tokenize>", lnum, pos, line))
indents = indents[:-1]
+
+ cur_indent = indents[-1]
+ while len(ctx) > 1 and ctx[-1][1] >= cur_indent:
+ ctx.pop()
+
yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
else: # continued statement
@@ -572,10 +581,16 @@ def _tokenize(readline, encoding):
(initial == '.' and token != '.' and token != '...')):
yield TokenInfo(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
+ if stashed:
+ yield stashed
+ stashed = None
yield TokenInfo(NL if parenlev > 0 else NEWLINE,
token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
+ if stashed:
+ yield stashed
+ stashed = None
yield TokenInfo(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = _compile(endpats[token])
@@ -603,7 +618,37 @@ def _tokenize(readline, encoding):
else: # ordinary string
yield TokenInfo(STRING, token, spos, epos, line)
elif initial.isidentifier(): # ordinary name
- yield TokenInfo(NAME, token, spos, epos, line)
+ if token in ('async', 'await'):
+ if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+ yield TokenInfo(
+ ASYNC if token == 'async' else AWAIT,
+ token, spos, epos, line)
+ continue
+
+ tok = TokenInfo(NAME, token, spos, epos, line)
+ if token == 'async' and not stashed:
+ stashed = tok
+ continue
+
+ if token == 'def':
+ if (stashed
+ and stashed.type == NAME
+ and stashed.string == 'async'):
+
+ ctx.append(('async', indents[-1]))
+
+ yield TokenInfo(ASYNC, stashed.string,
+ stashed.start, stashed.end,
+ stashed.line)
+ stashed = None
+ else:
+ ctx.append(('sync', indents[-1]))
+
+ if stashed:
+ yield stashed
+ stashed = None
+
+ yield tok
elif initial == '\\': # continued stmt
continued = 1
else:
@@ -611,12 +656,19 @@ def _tokenize(readline, encoding):
parenlev += 1
elif initial in ')]}':
parenlev -= 1
+ if stashed:
+ yield stashed
+ stashed = None
yield TokenInfo(OP, token, spos, epos, line)
else:
yield TokenInfo(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos += 1
+ if stashed:
+ yield stashed
+ stashed = None
+
for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
diff --git a/Lib/types.py b/Lib/types.py
index 4fb2def..49e4d04 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -43,6 +43,30 @@ MemberDescriptorType = type(FunctionType.__globals__)
del sys, _f, _g, _C, # Not for export
+_CO_GENERATOR = 0x20
+_CO_ITERABLE_COROUTINE = 0x100
+
+def coroutine(func):
+ """Convert regular generator function to a coroutine."""
+
+ # TODO: Implement this in C.
+
+ if (not isinstance(func, (FunctionType, MethodType)) or
+ not isinstance(getattr(func, '__code__', None), CodeType) or
+ not (func.__code__.co_flags & _CO_GENERATOR)):
+ raise TypeError('coroutine() expects a generator function')
+
+ co = func.__code__
+ func.__code__ = CodeType(
+ co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize,
+ co.co_flags | _CO_ITERABLE_COROUTINE,
+ co.co_code,
+ co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name,
+ co.co_firstlineno, co.co_lnotab, co.co_freevars, co.co_cellvars)
+
+ return func
+
+
# Provide a PEP 3115 compliant mechanism for class creation
def new_class(name, bases=(), kwds=None, exec_body=None):
"""Create a class object dynamically using the appropriate metaclass."""
diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py
index d798624..444f0b2 100644
--- a/Lib/xml/dom/xmlbuilder.py
+++ b/Lib/xml/dom/xmlbuilder.py
@@ -1,6 +1,7 @@
"""Implementation of the DOM Level 3 'LS-Load' feature."""
import copy
+import warnings
import xml.dom
from xml.dom.NodeFilter import NodeFilter
@@ -331,13 +332,33 @@ class DOMBuilderFilter:
del NodeFilter
+class _AsyncDeprecatedProperty:
+ def warn(self, cls):
+ clsname = cls.__name__
+ warnings.warn(
+ "{cls}.async is deprecated; use {cls}.async_".format(cls=clsname),
+ DeprecationWarning)
+
+ def __get__(self, instance, cls):
+ self.warn(cls)
+ if instance is not None:
+ return instance.async_
+ return False
+
+ def __set__(self, instance, value):
+ self.warn(type(instance))
+ setattr(instance, 'async_', value)
+
+
class DocumentLS:
"""Mixin to create documents that conform to the load/save spec."""
- async = False
+ async = _AsyncDeprecatedProperty()
+ async_ = False
def _get_async(self):
return False
+
def _set_async(self, async):
if async:
raise xml.dom.NotSupportedErr(
@@ -363,6 +384,9 @@ class DocumentLS:
return snode.toxml()
+del _AsyncDeprecatedProperty
+
+
class DOMImplementationLS:
MODE_SYNCHRONOUS = 1
MODE_ASYNCHRONOUS = 2