From 49c5da1d88f605248167f4d95b1dfe08c1f703c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Wed, 1 Mar 2006 22:49:05 +0000 Subject: Patch #1440601: Add col_offset attribute to AST nodes. --- Doc/lib/libast.tex | 8 ++- Include/Python-ast.h | 117 ++++++++++++++++++--------------- Include/node.h | 3 +- Lib/test/test_ast.py | 109 +++++++++++++++++++------------ Modules/parsermodule.c | 2 +- Parser/Python.asdl | 6 +- Parser/asdl.py | 2 + Parser/node.c | 3 +- Parser/parser.c | 14 ++-- Parser/parser.h | 2 +- Parser/parsetok.c | 9 ++- Parser/tokenizer.c | 5 ++ Parser/tokenizer.h | 1 + Python/Python-ast.c | 169 +++++++++++++++++++++++++++++++++--------------- Python/ast.c | 172 ++++++++++++++++++++++++++----------------------- Python/compile.c | 4 +- 16 files changed, 382 insertions(+), 244 deletions(-) diff --git a/Doc/lib/libast.tex b/Doc/lib/libast.tex index ca798c9..b3c3148 100644 --- a/Doc/lib/libast.tex +++ b/Doc/lib/libast.tex @@ -34,7 +34,13 @@ names of all child nodes. Each instance of a concrete class has one attribute for each child node, of the type as defined in the grammar. For example, \code{_ast.BinOp} -instances have an attribute \code{left} of type \code{_ast.expr}. +instances have an attribute \code{left} of type \code{_ast.expr}. +Instances of \code{_ast.expr} and \code{_ast.stmt} subclasses also +have lineno and col_offset attributes. The lineno is the line number +of source text (1 indexed so the first line is line 1) and the +col_offset is the utf8 byte offset of the first token that generated +the node. The utf8 offset is recorded because the parser uses utf8 +internally. If these attributes are marked as optional in the grammar (using a question mark), the value might be \code{None}. If the attributes diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 7aef19b..b3bc063 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -178,6 +178,7 @@ struct _stmt { } v; int lineno; + int col_offset; }; struct _expr { @@ -288,6 +289,7 @@ struct _expr { } v; int lineno; + int col_offset; }; struct _slice { @@ -346,68 +348,79 @@ mod_ty Interactive(asdl_seq * body, PyArena *arena); mod_ty Expression(expr_ty body, PyArena *arena); mod_ty Suite(asdl_seq * body, PyArena *arena); stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, - asdl_seq * decorators, int lineno, PyArena *arena); + asdl_seq * decorators, int lineno, int col_offset, PyArena + *arena); stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int - lineno, PyArena *arena); -stmt_ty Return(expr_ty value, int lineno, PyArena *arena); -stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena); -stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena); + lineno, int col_offset, PyArena *arena); +stmt_ty Return(expr_ty value, int lineno, int col_offset, PyArena *arena); +stmt_ty Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena); +stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, + PyArena *arena); stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, - PyArena *arena); -stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena - *arena); + int col_offset, PyArena *arena); +stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int + col_offset, PyArena *arena); stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, - int lineno, PyArena *arena); -stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, - PyArena *arena); -stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, - PyArena *arena); + int lineno, int col_offset, PyArena *arena); +stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int + col_offset, PyArena *arena); +stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int + col_offset, PyArena *arena); stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int - lineno, PyArena *arena); -stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena - *arena); + lineno, int col_offset, PyArena *arena); +stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int + col_offset, PyArena *arena); stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int - lineno, PyArena *arena); -stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena - *arena); -stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena); -stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena); + lineno, int col_offset, PyArena *arena); +stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int + col_offset, PyArena *arena); +stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena + *arena); +stmt_ty Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena); stmt_ty ImportFrom(identifier module, asdl_seq * names, int level, int lineno, - PyArena *arena); -stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena - *arena); -stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena); -stmt_ty Expr(expr_ty value, int lineno, PyArena *arena); -stmt_ty Pass(int lineno, PyArena *arena); -stmt_ty Break(int lineno, PyArena *arena); -stmt_ty Continue(int lineno, PyArena *arena); -expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena); -expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena - *arena); -expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena); -expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena); -expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena - *arena); -expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena); -expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena - *arena); -expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena - *arena); -expr_ty Yield(expr_ty value, int lineno, PyArena *arena); + int col_offset, PyArena *arena); +stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int + col_offset, PyArena *arena); +stmt_ty Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena); +stmt_ty Expr(expr_ty value, int lineno, int col_offset, PyArena *arena); +stmt_ty Pass(int lineno, int col_offset, PyArena *arena); +stmt_ty Break(int lineno, int col_offset, PyArena *arena); +stmt_ty Continue(int lineno, int col_offset, PyArena *arena); +expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, + PyArena *arena); +expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int + col_offset, PyArena *arena); +expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, + PyArena *arena); +expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, + PyArena *arena); +expr_ty IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int + col_offset, PyArena *arena); +expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, + PyArena *arena); +expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, int + col_offset, PyArena *arena); +expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int + col_offset, PyArena *arena); +expr_ty Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int - lineno, PyArena *arena); + lineno, int col_offset, PyArena *arena); expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty - starargs, expr_ty kwargs, int lineno, PyArena *arena); -expr_ty Repr(expr_ty value, int lineno, PyArena *arena); -expr_ty Num(object n, int lineno, PyArena *arena); -expr_ty Str(string s, int lineno, PyArena *arena); + starargs, expr_ty kwargs, int lineno, int col_offset, PyArena + *arena); +expr_ty Repr(expr_ty value, int lineno, int col_offset, PyArena *arena); +expr_ty Num(object n, int lineno, int col_offset, PyArena *arena); +expr_ty Str(string s, int lineno, int col_offset, PyArena *arena); expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int - lineno, PyArena *arena); + lineno, int col_offset, PyArena *arena); expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int - lineno, PyArena *arena); -expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena); -expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena); -expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena); + lineno, int col_offset, PyArena *arena); +expr_ty Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, + PyArena *arena); +expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, + PyArena *arena); +expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, + PyArena *arena); slice_ty Ellipsis(PyArena *arena); slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena); slice_ty ExtSlice(asdl_seq * dims, PyArena *arena); diff --git a/Include/node.h b/Include/node.h index 4c64708..e23e709 100644 --- a/Include/node.h +++ b/Include/node.h @@ -11,13 +11,14 @@ typedef struct _node { short n_type; char *n_str; int n_lineno; + int n_col_offset; int n_nchildren; struct _node *n_child; } node; PyAPI_FUNC(node *) PyNode_New(int type); PyAPI_FUNC(int) PyNode_AddChild(node *n, int type, - char *str, int lineno); + char *str, int lineno, int col_offset); PyAPI_FUNC(void) PyNode_Free(node *n); /* Node access functions */ diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 82bb140..0484513 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,4 +1,5 @@ import sys, itertools +import _ast def to_tuple(t): if t is None or isinstance(t, (basestring, int, long, complex)): @@ -6,6 +7,8 @@ def to_tuple(t): elif isinstance(t, list): return [to_tuple(e) for e in t] result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) if t._fields is None: return tuple(result) for f in t._fields: @@ -106,7 +109,10 @@ eval_tests = [ # List "[1,2,3]", # Tuple - "1,2,3" + "1,2,3", + # Combination + "a.b.c.d(a.b[1:2])", + ] # TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension @@ -121,58 +127,77 @@ if __name__=='__main__' and sys.argv[1:] == ['-g']: print "run_tests()" raise SystemExit +def test_order(ast_node, parent_pos): + + if not isinstance(ast_node, _ast.AST) or ast_node._fields == None: + return + if isinstance(ast_node, (_ast.expr, _ast.stmt)): + node_pos = (ast_node.lineno, ast_node.col_offset) + assert node_pos >= parent_pos, (node_pos, parent_pos) + parent_pos = (ast_node.lineno, ast_node.col_offset) + for name in ast_node._fields: + value = getattr(ast_node, name) + if isinstance(value, list): + for child in value: + test_order(child, parent_pos) + elif value != None: + test_order(value, parent_pos) + def run_tests(): for input, output, kind in ((exec_tests, exec_results, "exec"), (single_tests, single_results, "single"), (eval_tests, eval_results, "eval")): for i, o in itertools.izip(input, output): - assert to_tuple(compile(i, "?", kind, 0x400)) == o + ast_tree = compile(i, "?", kind, 0x400) + assert to_tuple(ast_tree) == o + test_order(ast_tree, (0, 0)) #### EVERYTHING BELOW IS GENERATED ##### exec_results = [ -('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Pass',)], [])]), -('Module', [('ClassDef', 'C', [], [('Pass',)])]), -('Module', [('FunctionDef', 'f', ('arguments', [], None, None, []), [('Return', ('Num', 1))], [])]), -('Module', [('Delete', [('Name', 'v', ('Del',))])]), -('Module', [('Assign', [('Name', 'v', ('Store',))], ('Num', 1))]), -('Module', [('AugAssign', ('Name', 'v', ('Load',)), ('Add',), ('Num', 1))]), -('Module', [('Print', ('Name', 'f', ('Load',)), [('Num', 1)], False)]), -('Module', [('For', ('Name', 'v', ('Store',)), ('Name', 'v', ('Load',)), [('Pass',)], [])]), -('Module', [('While', ('Name', 'v', ('Load',)), [('Pass',)], [])]), -('Module', [('If', ('Name', 'v', ('Load',)), [('Pass',)], [])]), -('Module', [('Raise', ('Name', 'Exception', ('Load',)), ('Str', 'string'), None)]), -('Module', [('TryExcept', [('Pass',)], [('excepthandler', ('Name', 'Exception', ('Load',)), None, [('Pass',)])], [])]), -('Module', [('TryFinally', [('Pass',)], [('Pass',)])]), -('Module', [('Assert', ('Name', 'v', ('Load',)), None)]), -('Module', [('Import', [('alias', 'sys', None)])]), -('Module', [('ImportFrom', 'sys', [('alias', 'v', None)], 0)]), -('Module', [('Exec', ('Str', 'v'), None, None)]), -('Module', [('Global', ['v'])]), -('Module', [('Expr', ('Num', 1))]), -('Module', [('Pass',)]), -('Module', [('Break',)]), -('Module', [('Continue',)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]), +('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]), +('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), +('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]), +('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Load',)), ('Add',), ('Num', (1, 5), 1))]), +('Module', [('Print', (1, 0), ('Name', (1, 8), 'f', ('Load',)), [('Num', (1, 11), 1)], False)]), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), +('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), +('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), +('Module', [('Raise', (1, 0), ('Name', (1, 6), 'Exception', ('Load',)), ('Str', (1, 17), 'string'), None)]), +('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('excepthandler', ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), +('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), +('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), +('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), +('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), +('Module', [('Exec', (1, 0), ('Str', (1, 5), 'v'), None, None)]), +('Module', [('Global', (1, 0), ['v'])]), +('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]), +('Module', [('Pass', (1, 0))]), +('Module', [('Break', (1, 0))]), +('Module', [('Continue', (1, 0))]), ] single_results = [ -('Interactive', [('Expr', ('BinOp', ('Num', 1), ('Add',), ('Num', 2)))]), +('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]), ] eval_results = [ -('Expression', ('BoolOp', ('And',), [('Name', 'a', ('Load',)), ('Name', 'b', ('Load',))])), -('Expression', ('BinOp', ('Name', 'a', ('Load',)), ('Add',), ('Name', 'b', ('Load',)))), -('Expression', ('UnaryOp', ('Not',), ('Name', 'v', ('Load',)))), -('Expression', ('Lambda', ('arguments', [], None, None, []), ('Name', 'None', ('Load',)))), -('Expression', ('Dict', [('Num', 1)], [('Num', 2)])), -('Expression', ('ListComp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])), -('Expression', ('GeneratorExp', ('Name', 'a', ('Load',)), [('comprehension', ('Name', 'b', ('Store',)), ('Name', 'c', ('Load',)), [('Name', 'd', ('Load',))])])), -('Expression', ('Compare', ('Num', 1), [('Lt',), ('Lt',)], [('Num', 2), ('Num', 3)])), -('Expression', ('Call', ('Name', 'f', ('Load',)), [('Num', 1), ('Num', 2)], [('keyword', 'c', ('Num', 3))], ('Name', 'd', ('Load',)), ('Name', 'e', ('Load',)))), -('Expression', ('Repr', ('Name', 'v', ('Load',)))), -('Expression', ('Num', 10L)), -('Expression', ('Str', 'string')), -('Expression', ('Attribute', ('Name', 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', ('Name', 'a', ('Load',)), ('Slice', ('Name', 'b', ('Load',)), ('Name', 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', 'v', ('Load',))), -('Expression', ('List', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))), -('Expression', ('Tuple', [('Num', 1), ('Num', 2), ('Num', 3)], ('Load',))), +('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))), +('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])), +('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), +('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), +('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])), +('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), +('Expression', ('Repr', (1, 0), ('Name', (1, 1), 'v', ('Load',)))), +('Expression', ('Num', (1, 0), 10L)), +('Expression', ('Str', (1, 0), 'string')), +('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0), 'v', ('Load',))), +('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))), +('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))), +('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)), ] run_tests() diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 0dcb50f..83165ba 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -715,7 +715,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num) Py_XDECREF(elem); return (0); } - err = PyNode_AddChild(root, type, strn, *line_num); + err = PyNode_AddChild(root, type, strn, *line_num, 0); if (err == E_NOMEM) { PyMem_DEL(strn); return (node *) PyErr_NoMemory(); diff --git a/Parser/Python.asdl b/Parser/Python.asdl index bcb5ecb..4397d89 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -46,7 +46,8 @@ module Python version "$Revision$" | Pass | Break | Continue -- XXX Jython will be different - attributes (int lineno) + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) -- BoolOp() can use left & right? expr = BoolOp(boolop op, expr* values) @@ -76,7 +77,8 @@ module Python version "$Revision$" | List(expr* elts, expr_context ctx) | Tuple(expr *elts, expr_context ctx) - attributes (int lineno) + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset) expr_context = Load | Store | Del | AugLoad | AugStore | Param diff --git a/Parser/asdl.py b/Parser/asdl.py index 3a69308..bd892b6 100644 --- a/Parser/asdl.py +++ b/Parser/asdl.py @@ -156,6 +156,8 @@ class ASDLParser(spark.GenericParser, object): if id.value != "attributes": raise ASDLSyntaxError(id.lineno, msg="expected attributes, found %s" % id) + if attributes: + attributes.reverse() return Sum(sum, attributes) def p_product(self, (_0, fields, _1)): diff --git a/Parser/node.c b/Parser/node.c index 75900ce..7ed6c0e 100644 --- a/Parser/node.c +++ b/Parser/node.c @@ -76,7 +76,7 @@ fancy_roundup(int n) int -PyNode_AddChild(register node *n1, int type, char *str, int lineno) +PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset) { const int nch = n1->n_nchildren; int current_capacity; @@ -103,6 +103,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno) n->n_type = type; n->n_str = str; n->n_lineno = lineno; + n->n_col_offset = col_offset; n->n_nchildren = 0; n->n_child = NULL; return 0; diff --git a/Parser/parser.c b/Parser/parser.c index 4a5307c..ada6be2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -105,11 +105,11 @@ PyParser_Delete(parser_state *ps) /* PARSER STACK OPERATIONS */ static int -shift(register stack *s, int type, char *str, int newstate, int lineno) +shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset) { int err; assert(!s_empty(s)); - err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno); + err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -117,13 +117,13 @@ shift(register stack *s, int type, char *str, int newstate, int lineno) } static int -push(register stack *s, int type, dfa *d, int newstate, int lineno) +push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset) { int err; register node *n; n = s->s_top->s_parent; assert(!s_empty(s)); - err = PyNode_AddChild(n, type, (char *)NULL, lineno); + err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset); if (err) return err; s->s_top->s_state = newstate; @@ -213,7 +213,7 @@ future_hack(parser_state *ps) int PyParser_AddToken(register parser_state *ps, register int type, char *str, - int lineno, int *expected_ret) + int lineno, int col_offset, int *expected_ret) { register int ilabel; int err; @@ -245,7 +245,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str, dfa *d1 = PyGrammar_FindDFA( ps->p_grammar, nt); if ((err = push(&ps->p_stack, nt, d1, - arrow, lineno)) > 0) { + arrow, lineno, col_offset)) > 0) { D(printf(" MemError: push\n")); return err; } @@ -255,7 +255,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str, /* Shift the token */ if ((err = shift(&ps->p_stack, type, str, - x, lineno)) > 0) { + x, lineno, col_offset)) > 0) { D(printf(" MemError: shift.\n")); return err; } diff --git a/Parser/parser.h b/Parser/parser.h index f5d2d0d..bdca3e9 100644 --- a/Parser/parser.h +++ b/Parser/parser.h @@ -32,7 +32,7 @@ typedef struct { parser_state *PyParser_New(grammar *g, int start); void PyParser_Delete(parser_state *ps); -int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, +int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset, int *expected_ret); void PyGrammar_AddAccelerators(grammar *g); diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 8b1f70c..77a2cac 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -130,6 +130,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, int type; size_t len; char *str; + int col_offset; type = PyTokenizer_Get(tok, &a, &b); if (type == ERRORTOKEN) { @@ -185,9 +186,13 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, len == 4 && str[0] == 'w' && strcmp(str, "with") == 0) handling_with = 1; #endif - + if (a >= tok->line_start) + col_offset = a - tok->line_start; + else + col_offset = -1; + if ((err_ret->error = - PyParser_AddToken(ps, (int)type, str, tok->lineno, + PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset, &(err_ret->expected))) != E_OK) { if (err_ret->error != E_DONE) PyObject_FREE(str); diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 036bed8..4174e9c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -764,6 +764,7 @@ tok_nextc(register struct tok_state *tok) } if (tok->start == NULL) tok->buf = tok->cur; + tok->line_start = tok->cur; tok->lineno++; tok->inp = end; return Py_CHARMASK(*tok->cur++); @@ -798,6 +799,7 @@ tok_nextc(register struct tok_state *tok) } tok->buf = buf; tok->cur = tok->buf + oldlen; + tok->line_start = tok->cur; strcpy(tok->buf + oldlen, new); PyMem_FREE(new); tok->inp = tok->buf + newlen; @@ -809,7 +811,9 @@ tok_nextc(register struct tok_state *tok) if (tok->buf != NULL) PyMem_DEL(tok->buf); tok->buf = new; + tok->line_start = tok->buf; tok->cur = tok->buf; + tok->line_start = tok->buf; tok->inp = strchr(tok->buf, '\0'); tok->end = tok->inp + 1; } @@ -877,6 +881,7 @@ tok_nextc(register struct tok_state *tok) done = tok->inp[-1] == '\n'; } tok->cur = tok->buf + cur; + tok->line_start = tok->cur; /* replace "\r\n" with "\n" */ /* For Mac we leave the \r, giving a syntax error */ pt = tok->inp - 2; diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 0b360617..5e7ebf7 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -45,6 +45,7 @@ struct tok_state { int read_coding_spec; /* whether 'coding:...' has been read */ char *encoding; int cont_line; /* whether we are in a continuation line. */ + const char* line_start; /* pointer to start of current line */ #ifndef PGEN PyObject *decoding_readline; /* codecs.open(...).readline */ PyObject *decoding_buffer; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f2dc346..3e49212 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -25,6 +25,7 @@ static char *Suite_fields[]={ static PyTypeObject *stmt_type; static char *stmt_attributes[] = { "lineno", + "col_offset", }; static PyObject* ast2obj_stmt(void*); static PyTypeObject *FunctionDef_type; @@ -142,6 +143,7 @@ static PyTypeObject *Continue_type; static PyTypeObject *expr_type; static char *expr_attributes[] = { "lineno", + "col_offset", }; static PyObject* ast2obj_expr(void*); static PyTypeObject *BoolOp_type; @@ -450,7 +452,7 @@ static int init_types(void) if (!Suite_type) return 0; stmt_type = make_type("stmt", AST_type, NULL, 0); if (!stmt_type) return 0; - if (!add_attributes(stmt_type, stmt_attributes, 1)) return 0; + if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields, 4); if (!FunctionDef_type) return 0; @@ -502,7 +504,7 @@ static int init_types(void) if (!Continue_type) return 0; expr_type = make_type("expr", AST_type, NULL, 0); if (!expr_type) return 0; - if (!add_attributes(expr_type, expr_attributes, 1)) return 0; + if (!add_attributes(expr_type, expr_attributes, 2)) return 0; BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2); if (!BoolOp_type) return 0; BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3); @@ -783,7 +785,7 @@ Suite(asdl_seq * body, PyArena *arena) stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * - decorators, int lineno, PyArena *arena) + decorators, int lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -807,12 +809,13 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * p->v.FunctionDef.body = body; p->v.FunctionDef.decorators = decorators; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, - PyArena *arena) +ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -830,11 +833,12 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int lineno, p->v.ClassDef.bases = bases; p->v.ClassDef.body = body; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Return(expr_ty value, int lineno, PyArena *arena) +Return(expr_ty value, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -845,11 +849,12 @@ Return(expr_ty value, int lineno, PyArena *arena) p->kind = Return_kind; p->v.Return.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Delete(asdl_seq * targets, int lineno, PyArena *arena) +Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -860,11 +865,13 @@ Delete(asdl_seq * targets, int lineno, PyArena *arena) p->kind = Delete_kind; p->v.Delete.targets = targets; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena) +Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena + *arena) { stmt_ty p; if (!value) { @@ -881,12 +888,13 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena) p->v.Assign.targets = targets; p->v.Assign.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena - *arena) +AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -914,11 +922,13 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, PyArena p->v.AugAssign.op = op; p->v.AugAssign.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena) +Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, int col_offset, + PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -931,12 +941,13 @@ Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena *arena) p->v.Print.values = values; p->v.Print.nl = nl; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int - lineno, PyArena *arena) + lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!target) { @@ -960,12 +971,13 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int p->v.For.body = body; p->v.For.orelse = orelse; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena - *arena) +While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -983,11 +995,13 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena p->v.While.body = body; p->v.While.orelse = orelse; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena) +If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -1005,12 +1019,13 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, PyArena *arena) p->v.If.body = body; p->v.If.orelse = orelse; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno, - PyArena *arena) + int col_offset, PyArena *arena) { stmt_ty p; if (!context_expr) { @@ -1028,11 +1043,13 @@ With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno, p->v.With.optional_vars = optional_vars; p->v.With.body = body; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena) +Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, int col_offset, + PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1045,12 +1062,13 @@ Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena *arena) p->v.Raise.inst = inst; p->v.Raise.tback = tback; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno, - PyArena *arena) + int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1063,11 +1081,13 @@ TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno, p->v.TryExcept.handlers = handlers; p->v.TryExcept.orelse = orelse; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena) +TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset, + PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1079,11 +1099,12 @@ TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena *arena) p->v.TryFinally.body = body; p->v.TryFinally.finalbody = finalbody; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena) +Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!test) { @@ -1100,11 +1121,12 @@ Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena) p->v.Assert.test = test; p->v.Assert.msg = msg; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Import(asdl_seq * names, int lineno, PyArena *arena) +Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1115,12 +1137,13 @@ Import(asdl_seq * names, int lineno, PyArena *arena) p->kind = Import_kind; p->v.Import.names = names; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena - *arena) +ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!module) { @@ -1138,11 +1161,13 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, PyArena p->v.ImportFrom.names = names; p->v.ImportFrom.level = level; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena) +Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, int col_offset, + PyArena *arena) { stmt_ty p; if (!body) { @@ -1160,11 +1185,12 @@ Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena *arena) p->v.Exec.globals = globals; p->v.Exec.locals = locals; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Global(asdl_seq * names, int lineno, PyArena *arena) +Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1175,11 +1201,12 @@ Global(asdl_seq * names, int lineno, PyArena *arena) p->kind = Global_kind; p->v.Global.names = names; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Expr(expr_ty value, int lineno, PyArena *arena) +Expr(expr_ty value, int lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!value) { @@ -1195,11 +1222,12 @@ Expr(expr_ty value, int lineno, PyArena *arena) p->kind = Expr_kind; p->v.Expr.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Pass(int lineno, PyArena *arena) +Pass(int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1209,11 +1237,12 @@ Pass(int lineno, PyArena *arena) } p->kind = Pass_kind; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Break(int lineno, PyArena *arena) +Break(int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1223,11 +1252,12 @@ Break(int lineno, PyArena *arena) } p->kind = Break_kind; p->lineno = lineno; + p->col_offset = col_offset; return p; } stmt_ty -Continue(int lineno, PyArena *arena) +Continue(int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1237,11 +1267,13 @@ Continue(int lineno, PyArena *arena) } p->kind = Continue_kind; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena) +BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!op) { @@ -1258,11 +1290,13 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena) p->v.BoolOp.op = op; p->v.BoolOp.values = values; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena) +BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset, + PyArena *arena) { expr_ty p; if (!left) { @@ -1290,11 +1324,13 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena *arena) p->v.BinOp.op = op; p->v.BinOp.right = right; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena) +UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!op) { @@ -1316,11 +1352,13 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena) p->v.UnaryOp.op = op; p->v.UnaryOp.operand = operand; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena) +Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!args) { @@ -1342,11 +1380,13 @@ Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena) p->v.Lambda.args = args; p->v.Lambda.body = body; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena) +IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset, + PyArena *arena) { expr_ty p; if (!test) { @@ -1374,11 +1414,13 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, PyArena *arena) p->v.IfExp.body = body; p->v.IfExp.orelse = orelse; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena) +Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena + *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1390,11 +1432,13 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena) p->v.Dict.keys = keys; p->v.Dict.values = values; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena) +ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, + PyArena *arena) { expr_ty p; if (!elt) { @@ -1411,11 +1455,13 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena) p->v.ListComp.elt = elt; p->v.ListComp.generators = generators; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena) +GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, + PyArena *arena) { expr_ty p; if (!elt) { @@ -1432,11 +1478,12 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena *arena) p->v.GeneratorExp.elt = elt; p->v.GeneratorExp.generators = generators; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Yield(expr_ty value, int lineno, PyArena *arena) +Yield(expr_ty value, int lineno, int col_offset, PyArena *arena) { expr_ty p; p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -1447,12 +1494,13 @@ Yield(expr_ty value, int lineno, PyArena *arena) p->kind = Yield_kind; p->v.Yield.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, - PyArena *arena) +Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, int + col_offset, PyArena *arena) { expr_ty p; if (!left) { @@ -1470,12 +1518,13 @@ Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int lineno, p->v.Compare.ops = ops; p->v.Compare.comparators = comparators; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, - expr_ty kwargs, int lineno, PyArena *arena) + expr_ty kwargs, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!func) { @@ -1495,11 +1544,12 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, p->v.Call.starargs = starargs; p->v.Call.kwargs = kwargs; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Repr(expr_ty value, int lineno, PyArena *arena) +Repr(expr_ty value, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -1515,11 +1565,12 @@ Repr(expr_ty value, int lineno, PyArena *arena) p->kind = Repr_kind; p->v.Repr.value = value; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Num(object n, int lineno, PyArena *arena) +Num(object n, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!n) { @@ -1535,11 +1586,12 @@ Num(object n, int lineno, PyArena *arena) p->kind = Num_kind; p->v.Num.n = n; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Str(string s, int lineno, PyArena *arena) +Str(string s, int lineno, int col_offset, PyArena *arena) { expr_ty p; if (!s) { @@ -1555,12 +1607,13 @@ Str(string s, int lineno, PyArena *arena) p->kind = Str_kind; p->v.Str.s = s; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, - PyArena *arena) +Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int + col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -1588,12 +1641,13 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, p->v.Attribute.attr = attr; p->v.Attribute.ctx = ctx; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, - PyArena *arena) +Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int + col_offset, PyArena *arena) { expr_ty p; if (!value) { @@ -1621,11 +1675,13 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, p->v.Subscript.slice = slice; p->v.Subscript.ctx = ctx; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena) +Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!id) { @@ -1647,11 +1703,13 @@ Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena) p->v.Name.id = id; p->v.Name.ctx = ctx; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena) +List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!ctx) { @@ -1668,11 +1726,13 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena) p->v.List.elts = elts; p->v.List.ctx = ctx; p->lineno = lineno; + p->col_offset = col_offset; return p; } expr_ty -Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena) +Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena + *arena) { expr_ty p; if (!ctx) { @@ -1689,6 +1749,7 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena) p->v.Tuple.elts = elts; p->v.Tuple.ctx = ctx; p->lineno = lineno; + p->col_offset = col_offset; return p; } @@ -2264,6 +2325,9 @@ ast2obj_stmt(void* _o) value = ast2obj_int(o->lineno); if (!value) goto failed; PyObject_SetAttrString(result, "lineno", value); + value = ast2obj_int(o->col_offset); + if (!value) goto failed; + PyObject_SetAttrString(result, "col_offset", value); return result; failed: Py_XDECREF(value); @@ -2580,6 +2644,9 @@ ast2obj_expr(void* _o) value = ast2obj_int(o->lineno); if (!value) goto failed; PyObject_SetAttrString(result, "lineno", value); + value = ast2obj_int(o->col_offset); + if (!value) goto failed; + PyObject_SetAttrString(result, "col_offset", value); return result; failed: Py_XDECREF(value); diff --git a/Python/ast.c b/Python/ast.c index 4e508eb..5fbea39 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -243,7 +243,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename, stmts = asdl_seq_new(1, arena); if (!stmts) goto error; - asdl_seq_SET(stmts, 0, Pass(n->n_lineno, arena)); + asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, arena)); return Interactive(stmts, arena); } else { @@ -564,7 +564,7 @@ compiler_complex_args(struct compiling *c, const node *n) ast_error(child, "assignment to None"); return NULL; } - arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), + arg = Name(NEW_IDENTIFIER(child), Store, LINENO(child), child->n_col_offset, c->c_arena); } else { @@ -573,7 +573,7 @@ compiler_complex_args(struct compiling *c, const node *n) asdl_seq_SET(args, i, arg); } - result = Tuple(args, Store, LINENO(n), c->c_arena); + result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena); if (!set_context(result, Store, n)) return NULL; return result; @@ -651,7 +651,7 @@ ast_for_arguments(struct compiling *c, const node *n) goto error; } name = Name(NEW_IDENTIFIER(CHILD(ch, 0)), - Param, LINENO(ch), c->c_arena); + Param, LINENO(ch), ch->n_col_offset, c->c_arena); if (!name) goto error; asdl_seq_SET(args, k++, name); @@ -696,14 +696,18 @@ ast_for_dotted_name(struct compiling *c, const node *n) { expr_ty e; identifier id; + int lineno, col_offset; int i; REQ(n, dotted_name); - + + lineno = LINENO(n); + col_offset = n->n_col_offset; + id = NEW_IDENTIFIER(CHILD(n, 0)); if (!id) return NULL; - e = Name(id, Load, LINENO(n), c->c_arena); + e = Name(id, Load, lineno, col_offset, c->c_arena); if (!e) return NULL; @@ -711,7 +715,7 @@ ast_for_dotted_name(struct compiling *c, const node *n) id = NEW_IDENTIFIER(CHILD(n, i)); if (!id) return NULL; - e = Attribute(e, id, Load, LINENO(CHILD(n, i)), c->c_arena); + e = Attribute(e, id, Load, lineno, col_offset, c->c_arena); if (!e) return NULL; } @@ -739,7 +743,7 @@ ast_for_decorator(struct compiling *c, const node *n) name_expr = NULL; } else if (NCH(n) == 5) { /* Call with no arguments */ - d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena); + d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); if (!d) return NULL; name_expr = NULL; @@ -811,7 +815,7 @@ ast_for_funcdef(struct compiling *c, const node *n) if (!body) return NULL; - return FunctionDef(name, args, body, decorator_seq, LINENO(n), c->c_arena); + return FunctionDef(name, args, body, decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); } static expr_ty @@ -838,7 +842,7 @@ ast_for_lambdef(struct compiling *c, const node *n) return NULL; } - return Lambda(args, expression, LINENO(n), c->c_arena); + return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena); } static expr_ty @@ -857,7 +861,7 @@ ast_for_ifexpr(struct compiling *c, const node *n) orelse = ast_for_expr(c, CHILD(n, 4)); if (!orelse) return NULL; - return IfExp(expression, body, orelse, LINENO(n), c->c_arena); + return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset, c->c_arena); } /* Count the number of 'for' loop in a list comprehension. @@ -968,7 +972,7 @@ ast_for_listcomp(struct compiling *c, const node *n) lc = comprehension(asdl_seq_GET(t, 0), expression, NULL, c->c_arena); else - lc = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena), + lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena), expression, NULL, c->c_arena); if (!lc) return NULL; @@ -1003,7 +1007,7 @@ ast_for_listcomp(struct compiling *c, const node *n) asdl_seq_SET(listcomps, i, lc); } - return ListComp(elt, listcomps, LINENO(n), c->c_arena); + return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena); } /* @@ -1113,7 +1117,7 @@ ast_for_genexp(struct compiling *c, const node *n) ge = comprehension(asdl_seq_GET(t, 0), expression, NULL, c->c_arena); else - ge = comprehension(Tuple(t, Store, LINENO(ch), c->c_arena), + ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, c->c_arena), expression, NULL, c->c_arena); if (!ge) @@ -1152,7 +1156,7 @@ ast_for_genexp(struct compiling *c, const node *n) asdl_seq_SET(genexps, i, ge); } - return GeneratorExp(elt, genexps, LINENO(n), c->c_arena); + return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena); } static expr_ty @@ -1167,14 +1171,14 @@ ast_for_atom(struct compiling *c, const node *n) case NAME: /* All names start in Load context, but may later be changed. */ - return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), c->c_arena); + return Name(NEW_IDENTIFIER(ch), Load, LINENO(n), n->n_col_offset, c->c_arena); case STRING: { PyObject *str = parsestrplus(c, n); if (!str) return NULL; PyArena_AddPyObject(c->c_arena, str); - return Str(str, LINENO(n), c->c_arena); + return Str(str, LINENO(n), n->n_col_offset, c->c_arena); } case NUMBER: { PyObject *pynum = parsenumber(STR(ch)); @@ -1182,13 +1186,13 @@ ast_for_atom(struct compiling *c, const node *n) return NULL; PyArena_AddPyObject(c->c_arena, pynum); - return Num(pynum, LINENO(n), c->c_arena); + return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); } case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); if (TYPE(ch) == RPAR) - return Tuple(NULL, Load, LINENO(n), c->c_arena); + return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); @@ -1201,7 +1205,7 @@ ast_for_atom(struct compiling *c, const node *n) ch = CHILD(n, 1); if (TYPE(ch) == RSQB) - return List(NULL, Load, LINENO(n), c->c_arena); + return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); REQ(ch, listmaker); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { @@ -1209,7 +1213,7 @@ ast_for_atom(struct compiling *c, const node *n) if (!elts) return NULL; - return List(elts, Load, LINENO(n), c->c_arena); + return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena); } else return ast_for_listcomp(c, ch); @@ -1243,14 +1247,14 @@ ast_for_atom(struct compiling *c, const node *n) asdl_seq_SET(values, i / 4, expression); } - return Dict(keys, values, LINENO(n), c->c_arena); + return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena); } case BACKQUOTE: { /* repr */ expr_ty expression = ast_for_testlist(c, CHILD(n, 1)); if (!expression) return NULL; - return Repr(expression, LINENO(n), c->c_arena); + return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena); } default: PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch)); @@ -1353,7 +1357,7 @@ ast_for_binop(struct compiling *c, const node *n) if (!operator) return NULL; - result = BinOp(expr1, operator, expr2, LINENO(n), c->c_arena); + result = BinOp(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena); if (!result) return NULL; @@ -1371,7 +1375,7 @@ ast_for_binop(struct compiling *c, const node *n) return NULL; tmp_result = BinOp(result, operator, tmp, - LINENO(next_oper), c->c_arena); + LINENO(next_oper), next_oper->n_col_offset, c->c_arena); if (!tmp) return NULL; result = tmp_result; @@ -1389,13 +1393,13 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) REQ(n, trailer); if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) - return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), c->c_arena); + return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); else return ast_for_call(c, CHILD(n, 1), left_expr); } else if (TYPE(CHILD(n, 0)) == DOT ) { return Attribute(left_expr, NEW_IDENTIFIER(CHILD(n, 1)), Load, - LINENO(n), c->c_arena); + LINENO(n), n->n_col_offset, c->c_arena); } else { REQ(CHILD(n, 0), LSQB); @@ -1405,7 +1409,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) slice_ty slc = ast_for_slice(c, CHILD(n, 0)); if (!slc) return NULL; - return Subscript(left_expr, slc, Load, LINENO(n), c->c_arena); + return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, c->c_arena); } else { /* The grammar is ambiguous here. The ambiguity is resolved @@ -1430,7 +1434,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) } if (!simple) { return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(n), c->c_arena); + Load, LINENO(n), n->n_col_offset, c->c_arena); } /* extract Index values and put them in a Tuple */ elts = asdl_seq_new(asdl_seq_LEN(slices), c->c_arena); @@ -1439,11 +1443,11 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) assert(slc->kind == Index_kind && slc->v.Index.value); asdl_seq_SET(elts, j, slc->v.Index.value); } - e = Tuple(elts, Load, LINENO(n), c->c_arena); + e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena); if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), - Load, LINENO(n), c->c_arena); + Load, LINENO(n), n->n_col_offset, c->c_arena); } } } @@ -1468,13 +1472,15 @@ ast_for_power(struct compiling *c, const node *n) tmp = ast_for_trailer(c, ch, e); if (!tmp) return NULL; + tmp->lineno = e->lineno; + tmp->col_offset = e->col_offset; e = tmp; } if (TYPE(CHILD(n, NCH(n) - 1)) == factor) { expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); if (!f) return NULL; - tmp = BinOp(e, Pow, f, LINENO(n), c->c_arena); + tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena); if (!tmp) return NULL; e = tmp; @@ -1542,9 +1548,9 @@ ast_for_expr(struct compiling *c, const node *n) asdl_seq_SET(seq, i / 2, e); } if (!strcmp(STR(CHILD(n, 1)), "and")) - return BoolOp(And, seq, LINENO(n), c->c_arena); + return BoolOp(And, seq, LINENO(n), n->n_col_offset, c->c_arena); assert(!strcmp(STR(CHILD(n, 1)), "or")); - return BoolOp(Or, seq, LINENO(n), c->c_arena); + return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena); case not_test: if (NCH(n) == 1) { n = CHILD(n, 0); @@ -1555,7 +1561,7 @@ ast_for_expr(struct compiling *c, const node *n) if (!expression) return NULL; - return UnaryOp(Not, expression, LINENO(n), c->c_arena); + return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, c->c_arena); } case comparison: if (NCH(n) == 1) { @@ -1594,7 +1600,7 @@ ast_for_expr(struct compiling *c, const node *n) return NULL; } - return Compare(expression, ops, cmps, LINENO(n), c->c_arena); + return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset, c->c_arena); } break; @@ -1620,7 +1626,7 @@ ast_for_expr(struct compiling *c, const node *n) if (!exp) return NULL; } - return Yield(exp, LINENO(n), c->c_arena); + return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); } case factor: { expr_ty expression; @@ -1636,11 +1642,11 @@ ast_for_expr(struct compiling *c, const node *n) switch (TYPE(CHILD(n, 0))) { case PLUS: - return UnaryOp(UAdd, expression, LINENO(n), c->c_arena); + return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset, c->c_arena); case MINUS: - return UnaryOp(USub, expression, LINENO(n), c->c_arena); + return UnaryOp(USub, expression, LINENO(n), n->n_col_offset, c->c_arena); case TILDE: - return UnaryOp(Invert, expression, LINENO(n), c->c_arena); + return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, "unhandled factor: %d", TYPE(CHILD(n, 0))); @@ -1761,7 +1767,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) } } - return Call(func, args, keywords, vararg, kwarg, LINENO(n), c->c_arena); + return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena); } static expr_ty @@ -1787,7 +1793,7 @@ ast_for_testlist(struct compiling *c, const node* n) asdl_seq *tmp = seq_for_testlist(c, n); if (!tmp) return NULL; - return Tuple(tmp, Load, LINENO(n), c->c_arena); + return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); } } @@ -1841,7 +1847,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!e) return NULL; - return Expr(e, LINENO(n), c->c_arena); + return Expr(e, LINENO(n), n->n_col_offset, c->c_arena); } else if (TYPE(CHILD(n, 1)) == augassign) { expr_ty expr1, expr2; @@ -1851,7 +1857,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (TYPE(ch) == testlist) expr1 = ast_for_testlist(c, ch); else - expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch), + expr1 = Yield(ast_for_expr(c, CHILD(ch, 0)), LINENO(ch), n->n_col_offset, c->c_arena); if (!expr1) @@ -1883,7 +1889,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (TYPE(ch) == testlist) expr2 = ast_for_testlist(c, ch); else - expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), c->c_arena); + expr2 = Yield(ast_for_expr(c, ch), LINENO(ch), ch->n_col_offset, c->c_arena); if (!expr2) return NULL; @@ -1891,7 +1897,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) if (!operator) return NULL; - return AugAssign(expr1, operator, expr2, LINENO(n), c->c_arena); + return AugAssign(expr1, operator, expr2, LINENO(n), n->n_col_offset, c->c_arena); } else { int i; @@ -1929,7 +1935,7 @@ ast_for_expr_stmt(struct compiling *c, const node *n) expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, LINENO(n), c->c_arena); + return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena); } } @@ -1961,7 +1967,7 @@ ast_for_print_stmt(struct compiling *c, const node *n) asdl_seq_SET(seq, j, expression); } nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true; - return Print(dest, seq, nl, LINENO(n), c->c_arena); + return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena); } static asdl_seq * @@ -1998,7 +2004,7 @@ ast_for_del_stmt(struct compiling *c, const node *n) expr_list = ast_for_exprlist(c, CHILD(n, 1), Del); if (!expr_list) return NULL; - return Delete(expr_list, LINENO(n), c->c_arena); + return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -2020,32 +2026,32 @@ ast_for_flow_stmt(struct compiling *c, const node *n) ch = CHILD(n, 0); switch (TYPE(ch)) { case break_stmt: - return Break(LINENO(n), c->c_arena); + return Break(LINENO(n), n->n_col_offset, c->c_arena); case continue_stmt: - return Continue(LINENO(n), c->c_arena); + return Continue(LINENO(n), n->n_col_offset, c->c_arena); case yield_stmt: { /* will reduce to yield_expr */ expr_ty exp = ast_for_expr(c, CHILD(ch, 0)); if (!exp) return NULL; - return Expr(exp, LINENO(n), c->c_arena); + return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena); } case return_stmt: if (NCH(ch) == 1) - return Return(NULL, LINENO(n), c->c_arena); + return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena); else { expr_ty expression = ast_for_testlist(c, CHILD(ch, 1)); if (!expression) return NULL; - return Return(expression, LINENO(n), c->c_arena); + return Return(expression, LINENO(n), n->n_col_offset, c->c_arena); } case raise_stmt: if (NCH(ch) == 1) - return Raise(NULL, NULL, NULL, LINENO(n), c->c_arena); + return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); else if (NCH(ch) == 2) { expr_ty expression = ast_for_expr(c, CHILD(ch, 1)); if (!expression) return NULL; - return Raise(expression, NULL, NULL, LINENO(n), c->c_arena); + return Raise(expression, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(ch) == 4) { expr_ty expr1, expr2; @@ -2057,7 +2063,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) if (!expr2) return NULL; - return Raise(expr1, expr2, NULL, LINENO(n), c->c_arena); + return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(ch) == 6) { expr_ty expr1, expr2, expr3; @@ -2072,7 +2078,7 @@ ast_for_flow_stmt(struct compiling *c, const node *n) if (!expr3) return NULL; - return Raise(expr1, expr2, expr3, LINENO(n), c->c_arena); + return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset, c->c_arena); } default: PyErr_Format(PyExc_SystemError, @@ -2167,10 +2173,14 @@ ast_for_import_stmt(struct compiling *c, const node *n) import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | '(' import_as_names ')' | import_as_names) */ + int lineno; + int col_offset; int i; asdl_seq *aliases; REQ(n, import_stmt); + lineno = LINENO(n); + col_offset = n->n_col_offset; n = CHILD(n, 0); if (TYPE(n) == import_name) { n = CHILD(n, 1); @@ -2184,11 +2194,10 @@ ast_for_import_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(aliases, i / 2, import_alias); } - return Import(aliases, LINENO(n), c->c_arena); + return Import(aliases, lineno, col_offset, c->c_arena); } else if (TYPE(n) == import_from) { int n_children; - int lineno = LINENO(n); int idx, ndots = 0; alias_ty mod = NULL; identifier modname; @@ -2259,7 +2268,7 @@ ast_for_import_stmt(struct compiling *c, const node *n) modname = mod->name; else modname = new_identifier("", c->c_arena); - return ImportFrom(modname, aliases, ndots, lineno, + return ImportFrom(modname, aliases, ndots, lineno, col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -2286,7 +2295,7 @@ ast_for_global_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(s, i / 2, name); } - return Global(s, LINENO(n), c->c_arena); + return Global(s, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -2317,7 +2326,7 @@ ast_for_exec_stmt(struct compiling *c, const node *n) return NULL; } - return Exec(expr1, globals, locals, LINENO(n), c->c_arena); + return Exec(expr1, globals, locals, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -2329,7 +2338,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) expr_ty expression = ast_for_expr(c, CHILD(n, 1)); if (!expression) return NULL; - return Assert(expression, NULL, LINENO(n), c->c_arena); + return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(n) == 4) { expr_ty expr1, expr2; @@ -2341,7 +2350,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) if (!expr2) return NULL; - return Assert(expr1, expr2, LINENO(n), c->c_arena); + return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, "improper number of parts to 'assert' statement: %d", @@ -2436,7 +2445,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) if (!suite_seq) return NULL; - return If(expression, suite_seq, NULL, LINENO(n), c->c_arena); + return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena); } s = STR(CHILD(n, 4)); @@ -2458,7 +2467,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) if (!seq2) return NULL; - return If(expression, seq1, seq2, LINENO(n), c->c_arena); + return If(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena); } else if (s[2] == 'i') { int i, n_elif, has_else = 0; @@ -2491,7 +2500,7 @@ ast_for_if_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(orelse, 0, If(expression, seq1, seq2, - LINENO(CHILD(n, NCH(n) - 6)), + LINENO(CHILD(n, NCH(n) - 6)), CHILD(n, NCH(n) - 6)->n_col_offset, c->c_arena)); /* the just-created orelse handled the last elif */ n_elif--; @@ -2513,12 +2522,12 @@ ast_for_if_stmt(struct compiling *c, const node *n) asdl_seq_SET(new, 0, If(expression, suite_seq, orelse, - LINENO(CHILD(n, off)), c->c_arena)); + LINENO(CHILD(n, off)), CHILD(n, off)->n_col_offset, c->c_arena)); orelse = new; } return If(ast_for_expr(c, CHILD(n, 1)), ast_for_suite(c, CHILD(n, 3)), - orelse, LINENO(n), c->c_arena); + orelse, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -2542,7 +2551,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) suite_seq = ast_for_suite(c, CHILD(n, 3)); if (!suite_seq) return NULL; - return While(expression, suite_seq, NULL, LINENO(n), c->c_arena); + return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena); } else if (NCH(n) == 7) { expr_ty expression; @@ -2558,7 +2567,7 @@ ast_for_while_stmt(struct compiling *c, const node *n) if (!seq2) return NULL; - return While(expression, seq1, seq2, LINENO(n), c->c_arena); + return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena); } PyErr_Format(PyExc_SystemError, @@ -2588,7 +2597,7 @@ ast_for_for_stmt(struct compiling *c, const node *n) if (asdl_seq_LEN(_target) == 1) target = asdl_seq_GET(_target, 0); else - target = Tuple(_target, Store, LINENO(n), c->c_arena); + target = Tuple(_target, Store, LINENO(n), n->n_col_offset, c->c_arena); expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) @@ -2597,7 +2606,7 @@ ast_for_for_stmt(struct compiling *c, const node *n) if (!suite_seq) return NULL; - return For(target, expression, suite_seq, seq, LINENO(n), c->c_arena); + return For(target, expression, suite_seq, seq, LINENO(n), n->n_col_offset, c->c_arena); } static excepthandler_ty @@ -2711,7 +2720,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) asdl_seq_SET(handlers, i, e); } - except_st = TryExcept(body, handlers, orelse, LINENO(n), c->c_arena); + except_st = TryExcept(body, handlers, orelse, LINENO(n), n->n_col_offset, c->c_arena); if (!finally) return except_st; @@ -2725,7 +2734,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) /* must be a try ... finally (except clauses are in body, if any exist) */ assert(finally != NULL); - return TryFinally(body, finally, LINENO(n), c->c_arena); + return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena); } static expr_ty @@ -2765,7 +2774,8 @@ ast_for_with_stmt(struct compiling *c, const node *n) if (!suite_seq) { return NULL; } - return With(context_expr, optional_vars, suite_seq, LINENO(n), c->c_arena); + return With(context_expr, optional_vars, suite_seq, LINENO(n), + n->n_col_offset, c->c_arena); } static stmt_ty @@ -2785,7 +2795,7 @@ ast_for_classdef(struct compiling *c, const node *n) s = ast_for_suite(c, CHILD(n, 3)); if (!s) return NULL; - return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset, c->c_arena); } /* check for empty base list */ @@ -2793,7 +2803,7 @@ ast_for_classdef(struct compiling *c, const node *n) s = ast_for_suite(c, CHILD(n,5)); if (!s) return NULL; - return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL, s, LINENO(n), n->n_col_offset, c->c_arena); } @@ -2805,7 +2815,7 @@ ast_for_classdef(struct compiling *c, const node *n) s = ast_for_suite(c, CHILD(n, 6)); if (!s) return NULL; - return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n), + return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases, s, LINENO(n), n->n_col_offset, c->c_arena); } @@ -2835,7 +2845,7 @@ ast_for_stmt(struct compiling *c, const node *n) case del_stmt: return ast_for_del_stmt(c, n); case pass_stmt: - return Pass(LINENO(n), c->c_arena); + return Pass(LINENO(n), n->n_col_offset, c->c_arena); case flow_stmt: return ast_for_flow_stmt(c, n); case import_stmt: diff --git a/Python/compile.c b/Python/compile.c index abc488c..b8fd3d5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3635,7 +3635,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) switch (e->kind) { case Attribute_kind: auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr, - AugLoad, e->lineno, c->c_arena); + AugLoad, e->lineno, e->col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); @@ -3646,7 +3646,7 @@ compiler_augassign(struct compiler *c, stmt_ty s) break; case Subscript_kind: auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice, - AugLoad, e->lineno, c->c_arena); + AugLoad, e->lineno, e->col_offset, c->c_arena); if (auge == NULL) return 0; VISIT(c, expr, auge); -- cgit v0.12