diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/ast.py | 8 | ||||
| -rw-r--r-- | Lib/symbol.py | 4 | ||||
| -rw-r--r-- | Lib/test/test_asdl_parser.py | 3 | ||||
| -rw-r--r-- | Lib/test/test_ast.py | 131 | ||||
| -rw-r--r-- | Lib/test/test_type_comments.py | 295 | ||||
| -rw-r--r-- | Lib/token.py | 12 |
6 files changed, 380 insertions, 73 deletions
@@ -27,12 +27,16 @@ from _ast import * -def parse(source, filename='<unknown>', mode='exec'): +def parse(source, filename='<unknown>', mode='exec', *, type_comments=False): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). + Pass type_comments=True to get back type comments where the syntax allows. """ - return compile(source, filename, mode, PyCF_ONLY_AST) + flags = PyCF_ONLY_AST + if type_comments: + flags |= PyCF_TYPE_COMMENTS + return compile(source, filename, mode, flags) def literal_eval(node_or_string): diff --git a/Lib/symbol.py b/Lib/symbol.py index b3fa089..36e0eec 100644 --- a/Lib/symbol.py +++ b/Lib/symbol.py @@ -100,6 +100,10 @@ comp_if = 340 encoding_decl = 341 yield_expr = 342 yield_arg = 343 +func_body_suite = 344 +func_type_input = 345 +func_type = 346 +typelist = 347 #--end constants-- sym_name = {} diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py index 30e6466..9eaceec 100644 --- a/Lib/test/test_asdl_parser.py +++ b/Lib/test/test_asdl_parser.py @@ -117,7 +117,8 @@ class TestAsdlParser(unittest.TestCase): v = CustomVisitor() v.visit(self.types['mod']) - self.assertEqual(v.names_with_seq, ['Module', 'Interactive', 'Suite']) + self.assertEqual(v.names_with_seq, + ['Module', 'Module', 'Interactive', 'FunctionType', 'Suite']) if __name__ == '__main__': diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 09e425d..609c8b2 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -455,7 +455,7 @@ class AST_Tests(unittest.TestCase): def test_module(self): body = [ast.Num(42)] - x = ast.Module(body) + x = ast.Module(body, []) self.assertEqual(x.body, body) def test_nodeclasses(self): @@ -524,13 +524,13 @@ class AST_Tests(unittest.TestCase): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)]) + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "<test>", "exec") self.assertIn("but got <_ast.expr", str(cm.exception)) def test_invalid_identitifer(self): - m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))]) + m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], []) ast.fix_missing_locations(m) with self.assertRaises(TypeError) as cm: compile(m, "<test>", "exec") @@ -575,11 +575,11 @@ class ASTHelpers_Test(unittest.TestCase): self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))])" + "keywords=[]))], type_ignores=[])" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))])" + "Constant('and cheese')], []))], [])" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " @@ -588,7 +588,7 @@ class ASTHelpers_Test(unittest.TestCase): "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" ) def test_copy_location(self): @@ -617,7 +617,8 @@ class ASTHelpers_Test(unittest.TestCase): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " + "type_ignores=[])" ) def test_increment_lineno(self): @@ -760,7 +761,7 @@ class ASTHelpers_Test(unittest.TestCase): names=[ast.alias(name='sleep')], level=None, lineno=None, col_offset=None)] - mod = ast.Module(body) + mod = ast.Module(body, []) with self.assertRaises(ValueError) as cm: compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) @@ -770,7 +771,7 @@ class ASTHelpers_Test(unittest.TestCase): names=[ast.alias(name='sleep')], level=None, lineno=0, col_offset=0)] - mod = ast.Module(body) + mod = ast.Module(body, []) code = compile(mod, 'test', 'exec') ns = {} exec(code, ns) @@ -790,11 +791,11 @@ class ASTValidatorTests(unittest.TestCase): self.assertIn(msg, str(cm.exception)) def expr(self, node, msg=None, *, exc=ValueError): - mod = ast.Module([ast.Expr(node)]) + mod = ast.Module([ast.Expr(node)], []) self.mod(mod, msg, exc=exc) def stmt(self, stmt, msg=None): - mod = ast.Module([stmt]) + mod = ast.Module([stmt], []) self.mod(mod, msg) def test_module(self): @@ -1603,61 +1604,61 @@ def main(): raise SystemExit unittest.main() -#### EVERYTHING BELOW IS GENERATED ##### +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### exec_results = [ -('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))]), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None)]), -('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]), -('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])]), -('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None)]), -('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), -('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1))]), -('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)))]), -('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]), -('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]), -('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), -('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), -('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), -('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), -('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), -('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)]), -('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), -('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), -('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), -('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), -('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), -('Module', [('Global', (1, 0), ['v'])]), -('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))]), -('Module', [('Pass', (1, 0))]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]), -('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]), -('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]), -('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]), -('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]), -('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None)]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))])], [], None)]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))])], [], None)]), -('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))]), -('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))]), -('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]), -('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]), -('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]), -('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])]), -('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None)]), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None, None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None, None), []), [('Pass', (1, 17))], [], None, None)], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None), ('arg', (1, 9), 'b', None, None), ('arg', (1, 14), 'c', None, None), ('arg', (1, 22), 'd', None, None), ('arg', (1, 28), 'e', None, None)], ('arg', (1, 35), 'args', None, None), [('arg', (1, 41), 'f', None, None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None, None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None, None)], []), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])], []), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])], []), +('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])], []), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None, None)], []), +('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1), None)], []), +('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []), +('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [], None)], []), +('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], []), +('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], []), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))], None)], []), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))], None)], []), +('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)], []), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], []), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], []), +('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], []), +('Module', [('Import', (1, 0), [('alias', 'sys', None)])], []), +('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], []), +('Module', [('Global', (1, 0), ['v'])], []), +('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))], []), +('Module', [('Pass', (1, 0))], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [], None)], []), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [], None)], []), +('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [], None)], []), +('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []), +('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []), +('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))], None)], [], None, None)], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))], None)], [], None, None)], []), +('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))], []), +('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))], []), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], []), +('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), +('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []), +('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])], []), +('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None, None)], []), ] single_results = [ ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1), ('Add',), ('Constant', (1, 2), 2)))]), diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py new file mode 100644 index 0000000..3065ddc --- /dev/null +++ b/Lib/test/test_type_comments.py @@ -0,0 +1,295 @@ +import ast +import unittest + + +funcdef = """\ +def foo(): + # type: () -> int + pass + +def bar(): # type: () -> None + pass +""" + +asyncdef = """\ +async def foo(): + # type: () -> int + return await bar() + +async def bar(): # type: () -> int + return await bar() +""" + +redundantdef = """\ +def foo(): # type: () -> int + # type: () -> str + return '' +""" + +nonasciidef = """\ +def foo(): + # type: () -> àçčéñt + pass +""" + +forstmt = """\ +for a in []: # type: int + pass +""" + +withstmt = """\ +with context() as a: # type: int + pass +""" + +vardecl = """\ +a = 0 # type: int +""" + +ignores = """\ +def foo(): + pass # type: ignore + +def bar(): + x = 1 # type: ignore +""" + +# Test for long-form type-comments in arguments. A test function +# named 'fabvk' would have two positional args, a and b, plus a +# var-arg *v, plus a kw-arg **k. It is verified in test_longargs() +# that it has exactly these arguments, no more, no fewer. +longargs = """\ +def fa( + a = 1, # type: A +): + pass + +def fa( + a = 1 # type: A +): + pass + +def fab( + a, # type: A + b, # type: B +): + pass + +def fab( + a, # type: A + b # type: B +): + pass + +def fv( + *v, # type: V +): + pass + +def fv( + *v # type: V +): + pass + +def fk( + **k, # type: K +): + pass + +def fk( + **k # type: K +): + pass + +def fvk( + *v, # type: V + **k, # type: K +): + pass + +def fvk( + *v, # type: V + **k # type: K +): + pass + +def fav( + a, # type: A + *v, # type: V +): + pass + +def fav( + a, # type: A + *v # type: V +): + pass + +def fak( + a, # type: A + **k, # type: K +): + pass + +def fak( + a, # type: A + **k # type: K +): + pass + +def favk( + a, # type: A + *v, # type: V + **k, # type: K +): + pass + +def favk( + a, # type: A + *v, # type: V + **k # type: K +): + pass +""" + + +class TypeCommentTests(unittest.TestCase): + + def parse(self, source): + return ast.parse(source, type_comments=True) + + def classic_parse(self, source): + return ast.parse(source) + + def test_funcdef(self): + tree = self.parse(funcdef) + self.assertEqual(tree.body[0].type_comment, "() -> int") + self.assertEqual(tree.body[1].type_comment, "() -> None") + tree = self.classic_parse(funcdef) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + def test_asyncdef(self): + tree = self.parse(asyncdef) + self.assertEqual(tree.body[0].type_comment, "() -> int") + self.assertEqual(tree.body[1].type_comment, "() -> int") + tree = self.classic_parse(asyncdef) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + def test_redundantdef(self): + with self.assertRaisesRegex(SyntaxError, "^Cannot have two type comments on def"): + tree = self.parse(redundantdef) + + def test_nonasciidef(self): + tree = self.parse(nonasciidef) + self.assertEqual(tree.body[0].type_comment, "() -> àçčéñt") + + def test_forstmt(self): + tree = self.parse(forstmt) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(forstmt) + self.assertEqual(tree.body[0].type_comment, None) + + def test_withstmt(self): + tree = self.parse(withstmt) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(withstmt) + self.assertEqual(tree.body[0].type_comment, None) + + def test_vardecl(self): + tree = self.parse(vardecl) + self.assertEqual(tree.body[0].type_comment, "int") + tree = self.classic_parse(vardecl) + self.assertEqual(tree.body[0].type_comment, None) + + def test_ignores(self): + tree = self.parse(ignores) + self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5]) + tree = self.classic_parse(ignores) + self.assertEqual(tree.type_ignores, []) + + def test_longargs(self): + tree = self.parse(longargs) + for t in tree.body: + # The expected args are encoded in the function name + todo = set(t.name[1:]) + self.assertEqual(len(t.args.args), + len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) + self.assertTrue(t.name.startswith('f'), t.name) + for c in t.name[1:]: + todo.remove(c) + if c == 'v': + arg = t.args.vararg + elif c == 'k': + arg = t.args.kwarg + else: + assert 0 <= ord(c) - ord('a') < len(t.args.args) + arg = t.args.args[ord(c) - ord('a')] + self.assertEqual(arg.arg, c) # That's the argument name + self.assertEqual(arg.type_comment, arg.arg.upper()) + assert not todo + tree = self.classic_parse(longargs) + for t in tree.body: + for arg in t.args.args + [t.args.vararg, t.args.kwarg]: + if arg is not None: + self.assertIsNone(arg.type_comment, "%s(%s:%r)" % + (t.name, arg.arg, arg.type_comment)) + + def test_inappropriate_type_comments(self): + """Tests for inappropriately-placed type comments. + + These should be silently ignored with type comments off, + but raise SyntaxError with type comments on. + + This is not meant to be exhaustive. + """ + + def check_both_ways(source): + ast.parse(source, type_comments=False) + with self.assertRaises(SyntaxError): + ast.parse(source, type_comments=True) + + check_both_ways("pass # type: int\n") + check_both_ways("foo() # type: int\n") + check_both_ways("x += 1 # type: int\n") + check_both_ways("while True: # type: int\n continue\n") + check_both_ways("while True:\n continue # type: int\n") + check_both_ways("try: # type: int\n pass\nfinally:\n pass\n") + check_both_ways("try:\n pass\nfinally: # type: int\n pass\n") + + def test_func_type_input(self): + + def parse_func_type_input(source): + return ast.parse(source, "<unknown>", "func_type") + + # Some checks below will crash if the returned structure is wrong + tree = parse_func_type_input("() -> int") + self.assertEqual(tree.argtypes, []) + self.assertEqual(tree.returns.id, "int") + + tree = parse_func_type_input("(int) -> List[str]") + self.assertEqual(len(tree.argtypes), 1) + arg = tree.argtypes[0] + self.assertEqual(arg.id, "int") + self.assertEqual(tree.returns.value.id, "List") + self.assertEqual(tree.returns.slice.value.id, "str") + + tree = parse_func_type_input("(int, *str, **Any) -> float") + self.assertEqual(tree.argtypes[0].id, "int") + self.assertEqual(tree.argtypes[1].id, "str") + self.assertEqual(tree.argtypes[2].id, "Any") + self.assertEqual(tree.returns.id, "float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(int, *str, *Any) -> float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(int, **str, Any) -> float") + + with self.assertRaises(SyntaxError): + tree = parse_func_type_input("(**int, **str) -> float") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/token.py b/Lib/token.py index 7224eca..9bf80a5 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -58,12 +58,14 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 OP = 54 +TYPE_IGNORE = 55 +TYPE_COMMENT = 56 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 55 -COMMENT = 56 -NL = 57 -ENCODING = 58 -N_TOKENS = 59 +ERRORTOKEN = 57 +COMMENT = 58 +NL = 59 +ENCODING = 60 +N_TOKENS = 61 # Special definitions for cooperation with parser NT_OFFSET = 256 |