summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Doc/library/ast.rst40
-rw-r--r--Include/Python-ast.h274
-rw-r--r--Include/node.h6
-rw-r--r--Lib/ast.py100
-rw-r--r--Lib/test/test_asdl_parser.py6
-rw-r--r--Lib/test/test_ast.py354
-rw-r--r--Lib/test/test_parser.py2
-rw-r--r--Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst2
-rw-r--r--Modules/parsermodule.c2
-rw-r--r--Parser/Python.asdl8
-rw-r--r--Parser/asdl_c.py8
-rw-r--r--Parser/node.c26
-rw-r--r--Parser/parser.c22
-rw-r--r--Parser/parser.h4
-rw-r--r--Parser/parsetok.c14
-rw-r--r--Python/Python-ast.c578
-rw-r--r--Python/ast.c346
-rw-r--r--Python/ast_opt.c3
-rw-r--r--Python/compile.c6
19 files changed, 1406 insertions, 395 deletions
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index 2883f3c..7715a28 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -61,13 +61,21 @@ Node classes
.. attribute:: lineno
col_offset
+ end_lineno
+ end_col_offset
Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have
- :attr:`lineno` and :attr:`col_offset` attributes. The :attr:`lineno` is
- the line number of source text (1-indexed so the first line is line 1) and
- the :attr:`col_offset` is the UTF-8 byte offset of the first token that
- generated the node. The UTF-8 offset is recorded because the parser uses
- UTF-8 internally.
+ :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset`
+ attributes. The :attr:`lineno` and :attr:`end_lineno` are the first and
+ last line numbers of source text span (1-indexed so the first line is line 1)
+ and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding
+ UTF-8 byte offsets of the first and last tokens that generated the node.
+ The UTF-8 offset is recorded because the parser uses UTF-8 internally.
+
+ Note that the end positions are not required by the compiler and are
+ therefore optional. The end offset is *after* the last symbol, for example
+ one can get the source segment of a one-line expression node using
+ ``source_line[node.col_offset : node.end_col_offset]``.
The constructor of a class :class:`ast.T` parses its arguments as follows:
@@ -162,6 +170,18 @@ and classes for traversing abstract syntax trees:
:class:`AsyncFunctionDef` is now supported.
+.. function:: get_source_segment(source, node, *, padded=False)
+
+ Get source code segment of the *source* that generated *node*.
+ If some location information (:attr:`lineno`, :attr:`end_lineno`,
+ :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``.
+
+ If *padded* is ``True``, the first line of a multi-line statement will
+ be padded with spaces to match its original position.
+
+ .. versionadded:: 3.8
+
+
.. function:: fix_missing_locations(node)
When you compile a node tree with :func:`compile`, the compiler expects
@@ -173,14 +193,16 @@ and classes for traversing abstract syntax trees:
.. function:: increment_lineno(node, n=1)
- Increment the line number of each node in the tree starting at *node* by *n*.
- This is useful to "move code" to a different location in a file.
+ Increment the line number and end line number of each node in the tree
+ starting at *node* by *n*. This is useful to "move code" to a different
+ location in a file.
.. function:: copy_location(new_node, old_node)
- Copy source location (:attr:`lineno` and :attr:`col_offset`) from *old_node*
- to *new_node* if possible, and return *new_node*.
+ Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`,
+ and :attr:`end_col_offset`) from *old_node* to *new_node* if possible,
+ and return *new_node*.
.. function:: iter_fields(node)
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 1a2b829..f8394e6 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -210,6 +210,8 @@ struct _stmt {
} v;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
};
enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
@@ -353,6 +355,8 @@ struct _expr {
} v;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
};
enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3};
@@ -396,6 +400,8 @@ struct _excepthandler {
} v;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
};
struct _arguments {
@@ -412,6 +418,8 @@ struct _arg {
expr_ty annotation;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
};
struct _keyword {
@@ -430,6 +438,7 @@ struct _withitem {
};
+// Note: these macros affect function definitions, not only call sites.
#define Module(a0, a1) _Py_Module(a0, a1)
mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
#define Interactive(a0, a1) _Py_Interactive(a0, a1)
@@ -438,152 +447,188 @@ mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
mod_ty _Py_Expression(expr_ty body, PyArena *arena);
#define Suite(a0, a1) _Py_Suite(a0, a1)
mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
asdl_seq * decorator_list, expr_ty returns, int lineno,
- int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+ int col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
body, asdl_seq * decorator_list, expr_ty returns,
- int lineno, int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
+ int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
asdl_seq * body, asdl_seq * decorator_list, int lineno,
- int col_offset, PyArena *arena);
-#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
-stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
-stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena
- *arena);
-#define Assign(a0, a1, a2, a3, a4) _Py_Assign(a0, a1, a2, a3, a4)
+ int col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena);
+#define Return(a0, a1, a2, a3, a4, a5) _Py_Return(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena);
+#define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Assign(a0, a1, a2, a3, a4, a5, a6) _Py_Assign(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int
- col_offset, PyArena *arena);
-#define AugAssign(a0, a1, a2, a3, a4, a5) _Py_AugAssign(a0, a1, a2, a3, a4, a5)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
- lineno, int col_offset, PyArena *arena);
-#define AnnAssign(a0, a1, a2, a3, a4, a5, a6) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6)
+ lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int
- simple, int lineno, int col_offset, PyArena *arena);
-#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
+ simple, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
- orelse, int lineno, int col_offset, PyArena *arena);
-#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+ orelse, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
- orelse, int lineno, int col_offset, PyArena *arena);
-#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
+ orelse, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
- int col_offset, PyArena *arena);
-#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
+ int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define If(a0, a1, a2, a3, a4, a5, a6, a7) _Py_If(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
- int col_offset, PyArena *arena);
-#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
+ int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define With(a0, a1, a2, a3, a4, a5, a6) _Py_With(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
- PyArena *arena);
-#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+ int end_lineno, int end_col_offset, PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6)
stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
- col_offset, PyArena *arena);
-#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
-stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
- PyArena *arena);
-#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_Try(a0, a1, a2, a3, a4, a5, a6, a7, a8)
stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
- asdl_seq * finalbody, int lineno, int col_offset, PyArena
- *arena);
-#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4)
-stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset,
- PyArena *arena);
-#define Import(a0, a1, a2, a3) _Py_Import(a0, a1, a2, a3)
-stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, PyArena
- *arena);
-#define ImportFrom(a0, a1, a2, a3, a4, a5) _Py_ImportFrom(a0, a1, a2, a3, a4, a5)
+ asdl_seq * finalbody, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Assert(a0, a1, a2, a3, a4, a5, a6) _Py_Assert(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Import(a0, a1, a2, a3, a4, a5) _Py_Import(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7)
stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
- lineno, int col_offset, PyArena *arena);
-#define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
-stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
- *arena);
-#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3)
-stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena
- *arena);
-#define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
-stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)
-stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
-#define Break(a0, a1, a2) _Py_Break(a0, a1, a2)
-stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
-#define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
-stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
-#define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
+ lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Global(a0, a1, a2, a3, a4, a5) _Py_Global(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Nonlocal(a0, a1, a2, a3, a4, a5) _Py_Nonlocal(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Expr(a0, a1, a2, a3, a4, a5) _Py_Expr(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Pass(a0, a1, a2, a3, a4) _Py_Pass(a0, a1, a2, a3, a4)
+stmt_ty _Py_Pass(int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Break(a0, a1, a2, a3, a4) _Py_Break(a0, a1, a2, a3, a4)
+stmt_ty _Py_Break(int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Continue(a0, a1, a2, a3, a4) _Py_Continue(a0, a1, a2, a3, a4)
+stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
- PyArena *arena);
-#define BinOp(a0, a1, a2, a3, a4, a5) _Py_BinOp(a0, a1, a2, a3, a4, a5)
+ int end_lineno, int end_col_offset, PyArena *arena);
+#define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
- col_offset, PyArena *arena);
-#define UnaryOp(a0, a1, a2, a3, a4) _Py_UnaryOp(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define UnaryOp(a0, a1, a2, a3, a4, a5, a6) _Py_UnaryOp(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
- PyArena *arena);
-#define Lambda(a0, a1, a2, a3, a4) _Py_Lambda(a0, a1, a2, a3, a4)
+ int end_lineno, int end_col_offset, PyArena *arena);
+#define Lambda(a0, a1, a2, a3, a4, a5, a6) _Py_Lambda(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
- PyArena *arena);
-#define IfExp(a0, a1, a2, a3, a4, a5) _Py_IfExp(a0, a1, a2, a3, a4, a5)
+ int end_lineno, int end_col_offset, PyArena *arena);
+#define IfExp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_IfExp(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
- col_offset, PyArena *arena);
-#define Dict(a0, a1, a2, a3, a4) _Py_Dict(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define Dict(a0, a1, a2, a3, a4, a5, a6) _Py_Dict(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_Dict(asdl_seq * keys, asdl_seq * values, int lineno, int
- col_offset, PyArena *arena);
-#define Set(a0, a1, a2, a3) _Py_Set(a0, a1, a2, a3)
-expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
-#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define Set(a0, a1, a2, a3, a4, a5) _Py_Set(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena);
+#define ListComp(a0, a1, a2, a3, a4, a5, a6) _Py_ListComp(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
- col_offset, PyArena *arena);
-#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define SetComp(a0, a1, a2, a3, a4, a5, a6) _Py_SetComp(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
- col_offset, PyArena *arena);
-#define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define DictComp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_DictComp(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
- lineno, int col_offset, PyArena *arena);
-#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
+ lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define GeneratorExp(a0, a1, a2, a3, a4, a5, a6) _Py_GeneratorExp(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
- col_offset, PyArena *arena);
-#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
-expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
-expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)
-expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
- *arena);
-#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
+ col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena);
+#define Await(a0, a1, a2, a3, a4, a5) _Py_Await(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena);
+#define Yield(a0, a1, a2, a3, a4, a5) _Py_Yield(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena);
+#define YieldFrom(a0, a1, a2, a3, a4, a5) _Py_YieldFrom(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Compare(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Compare(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
- int lineno, int col_offset, PyArena *arena);
-#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5)
+ int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
- lineno, int col_offset, PyArena *arena);
-#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5)
+ lineno, int col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
- int lineno, int col_offset, PyArena *arena);
-#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3)
-expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
- *arena);
-#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3)
-expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena
- *arena);
-#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
+ int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Constant(a0, a1, a2, a3, a4, a5) _Py_Constant(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Constant(constant value, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena);
+#define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
- lineno, int col_offset, PyArena *arena);
-#define Subscript(a0, a1, a2, a3, a4, a5) _Py_Subscript(a0, a1, a2, a3, a4, a5)
+ lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7)
expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
- lineno, int col_offset, PyArena *arena);
-#define Starred(a0, a1, a2, a3, a4) _Py_Starred(a0, a1, a2, a3, a4)
+ lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena);
+#define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_Starred(expr_ty value, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena);
-#define Name(a0, a1, a2, a3, a4) _Py_Name(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define Name(a0, a1, a2, a3, a4, a5, a6) _Py_Name(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena);
-#define List(a0, a1, a2, a3, a4) _Py_List(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define List(a0, a1, a2, a3, a4, a5, a6) _Py_List(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena);
-#define Tuple(a0, a1, a2, a3, a4) _Py_Tuple(a0, a1, a2, a3, a4)
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
+#define Tuple(a0, a1, a2, a3, a4, a5, a6) _Py_Tuple(a0, a1, a2, a3, a4, a5, a6)
expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena);
+ col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena);
#define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3)
slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
#define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1)
@@ -593,17 +638,18 @@ slice_ty _Py_Index(expr_ty value, PyArena *arena);
#define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4)
comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
ifs, int is_async, PyArena *arena);
-#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5)
+#define ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7)
excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
- body, int lineno, int col_offset, PyArena
+ body, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena
*arena);
#define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
asdl_seq * defaults, PyArena *arena);
-#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4)
+#define arg(a0, a1, a2, a3, a4, a5, a6) _Py_arg(a0, a1, a2, a3, a4, a5, a6)
arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset,
- PyArena *arena);
+ int end_lineno, int end_col_offset, PyArena *arena);
#define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
#define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
diff --git a/Include/node.h b/Include/node.h
index 40596df..2b39074 100644
--- a/Include/node.h
+++ b/Include/node.h
@@ -14,11 +14,14 @@ typedef struct _node {
int n_col_offset;
int n_nchildren;
struct _node *n_child;
+ int n_end_lineno;
+ int n_end_col_offset;
} node;
PyAPI_FUNC(node *) PyNode_New(int type);
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
- char *str, int lineno, int col_offset);
+ char *str, int lineno, int col_offset,
+ int end_lineno, int end_col_offset);
PyAPI_FUNC(void) PyNode_Free(node *n);
#ifndef Py_LIMITED_API
PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
@@ -37,6 +40,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
#define REQ(n, type) assert(TYPE(n) == (type))
PyAPI_FUNC(void) PyNode_ListTree(node *);
+void _PyNode_FinalizeEndPos(node *n); // helper also used in parsetok.c
#ifdef __cplusplus
}
diff --git a/Lib/ast.py b/Lib/ast.py
index 03b8a1b..6c1e978 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -115,10 +115,10 @@ def dump(node, annotate_fields=True, include_attributes=False):
def copy_location(new_node, old_node):
"""
- Copy source location (`lineno` and `col_offset` attributes) from
- *old_node* to *new_node* if possible, and return *new_node*.
+ Copy source location (`lineno`, `col_offset`, `end_lineno`, and `end_col_offset`
+ attributes) from *old_node* to *new_node* if possible, and return *new_node*.
"""
- for attr in 'lineno', 'col_offset':
+ for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset':
if attr in old_node._attributes and attr in new_node._attributes \
and hasattr(old_node, attr):
setattr(new_node, attr, getattr(old_node, attr))
@@ -133,31 +133,44 @@ def fix_missing_locations(node):
recursively where not already set, by setting them to the values of the
parent node. It works recursively starting at *node*.
"""
- def _fix(node, lineno, col_offset):
+ def _fix(node, lineno, col_offset, end_lineno, end_col_offset):
if 'lineno' in node._attributes:
if not hasattr(node, 'lineno'):
node.lineno = lineno
else:
lineno = node.lineno
+ if 'end_lineno' in node._attributes:
+ if not hasattr(node, 'end_lineno'):
+ node.end_lineno = end_lineno
+ else:
+ end_lineno = node.end_lineno
if 'col_offset' in node._attributes:
if not hasattr(node, 'col_offset'):
node.col_offset = col_offset
else:
col_offset = node.col_offset
+ if 'end_col_offset' in node._attributes:
+ if not hasattr(node, 'end_col_offset'):
+ node.end_col_offset = end_col_offset
+ else:
+ end_col_offset = node.end_col_offset
for child in iter_child_nodes(node):
- _fix(child, lineno, col_offset)
- _fix(node, 1, 0)
+ _fix(child, lineno, col_offset, end_lineno, end_col_offset)
+ _fix(node, 1, 0, 1, 0)
return node
def increment_lineno(node, n=1):
"""
- Increment the line number of each node in the tree starting at *node* by *n*.
- This is useful to "move code" to a different location in a file.
+ Increment the line number and end line number of each node in the tree
+ starting at *node* by *n*. This is useful to "move code" to a different
+ location in a file.
"""
for child in walk(node):
if 'lineno' in child._attributes:
child.lineno = getattr(child, 'lineno', 0) + n
+ if 'end_lineno' in child._attributes:
+ child.end_lineno = getattr(child, 'end_lineno', 0) + n
return node
@@ -213,6 +226,77 @@ def get_docstring(node, clean=True):
return text
+def _splitlines_no_ff(source):
+ """Split a string into lines ignoring form feed and other chars.
+
+ This mimics how the Python parser splits source code.
+ """
+ idx = 0
+ lines = []
+ next_line = ''
+ while idx < len(source):
+ c = source[idx]
+ next_line += c
+ idx += 1
+ # Keep \r\n together
+ if c == '\r' and idx < len(source) and source[idx] == '\n':
+ next_line += '\n'
+ idx += 1
+ if c in '\r\n':
+ lines.append(next_line)
+ next_line = ''
+
+ if next_line:
+ lines.append(next_line)
+ return lines
+
+
+def _pad_whitespace(source):
+ """Replace all chars except '\f\t' in a line with spaces."""
+ result = ''
+ for c in source:
+ if c in '\f\t':
+ result += c
+ else:
+ result += ' '
+ return result
+
+
+def get_source_segment(source, node, *, padded=False):
+ """Get source code segment of the *source* that generated *node*.
+
+ If some location information (`lineno`, `end_lineno`, `col_offset`,
+ or `end_col_offset`) is missing, return None.
+
+ If *padded* is `True`, the first line of a multi-line statement will
+ be padded with spaces to match its original position.
+ """
+ try:
+ lineno = node.lineno - 1
+ end_lineno = node.end_lineno - 1
+ col_offset = node.col_offset
+ end_col_offset = node.end_col_offset
+ except AttributeError:
+ return None
+
+ lines = _splitlines_no_ff(source)
+ if end_lineno == lineno:
+ return lines[lineno].encode()[col_offset:end_col_offset].decode()
+
+ if padded:
+ padding = _pad_whitespace(lines[lineno].encode()[:col_offset].decode())
+ else:
+ padding = ''
+
+ first = padding + lines[lineno].encode()[col_offset:].decode()
+ last = lines[end_lineno].encode()[:end_col_offset].decode()
+ lines = lines[lineno+1:end_lineno]
+
+ lines.insert(0, first)
+ lines.append(last)
+ return ''.join(lines)
+
+
def walk(node):
"""
Recursively yield all descendant nodes in the tree starting at *node*
diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py
index 15bc684..30e6466 100644
--- a/Lib/test/test_asdl_parser.py
+++ b/Lib/test/test_asdl_parser.py
@@ -62,14 +62,16 @@ class TestAsdlParser(unittest.TestCase):
def test_attributes(self):
stmt = self.types['stmt']
- self.assertEqual(len(stmt.attributes), 2)
+ self.assertEqual(len(stmt.attributes), 4)
self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)')
self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)')
+ self.assertEqual(str(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
+ self.assertEqual(str(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
def test_constructor_fields(self):
ehandler = self.types['excepthandler']
self.assertEqual(len(ehandler.types), 1)
- self.assertEqual(len(ehandler.attributes), 2)
+ self.assertEqual(len(ehandler.attributes), 4)
cons = ehandler.types[0]
self.assertIsInstance(cons, self.asdl.Constructor)
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 4bf77ff..09e425d 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -4,6 +4,7 @@ import os
import sys
import unittest
import weakref
+from textwrap import dedent
from test import support
@@ -582,19 +583,22 @@ class ASTHelpers_Test(unittest.TestCase):
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
- "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
- "lineno=1, col_offset=5), Constant(value='and cheese', lineno=1, "
- "col_offset=11)], keywords=[], "
- "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), "
+ "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, "
+ "end_lineno=1, end_col_offset=9), Constant(value='and cheese', "
+ "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], "
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), "
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])"
)
def test_copy_location(self):
src = ast.parse('1 + 1', mode='eval')
src.body.right = ast.copy_location(ast.Num(2), src.body.right)
self.assertEqual(ast.dump(src, include_attributes=True),
- 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0), '
- 'op=Add(), right=Constant(value=2, lineno=1, col_offset=4), lineno=1, '
- 'col_offset=0))'
+ 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, '
+ 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, '
+ 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, '
+ 'col_offset=0, end_lineno=1, end_col_offset=5))'
)
def test_fix_missing_locations(self):
@@ -602,32 +606,37 @@ class ASTHelpers_Test(unittest.TestCase):
src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()),
[ast.Str('eggs')], [])))
self.assertEqual(src, ast.fix_missing_locations(src))
+ self.maxDiff = None
self.assertEqual(ast.dump(src, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
- "lineno=1, col_offset=0), args=[Constant(value='spam', lineno=1, "
- "col_offset=6)], keywords=[], "
- "lineno=1, col_offset=0), lineno=1, col_offset=0), "
- "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
- "col_offset=0), args=[Constant(value='eggs', lineno=1, col_offset=0)], "
- "keywords=[], lineno=1, "
- "col_offset=0), lineno=1, col_offset=0)])"
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), "
+ "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, "
+ "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
+ "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, "
+ "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), "
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), "
+ "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, "
+ "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
+ "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])"
)
def test_increment_lineno(self):
src = ast.parse('1 + 1', mode='eval')
self.assertEqual(ast.increment_lineno(src, n=3), src)
self.assertEqual(ast.dump(src, include_attributes=True),
- 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), '
- 'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, '
- 'col_offset=0))'
+ 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
+ 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
+ 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
+ 'col_offset=0, end_lineno=4, end_col_offset=5))'
)
# issue10869: do not increment lineno of root twice
src = ast.parse('1 + 1', mode='eval')
self.assertEqual(ast.increment_lineno(src.body, n=3), src.body)
self.assertEqual(ast.dump(src, include_attributes=True),
- 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0), '
- 'op=Add(), right=Constant(value=1, lineno=4, col_offset=4), lineno=4, '
- 'col_offset=0))'
+ 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, '
+ 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, '
+ 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, '
+ 'col_offset=0, end_lineno=4, end_col_offset=5))'
)
def test_iter_fields(self):
@@ -1274,6 +1283,311 @@ class ConstantTests(unittest.TestCase):
self.assertEqual(ast.literal_eval(binop), 10+20j)
+class EndPositionTests(unittest.TestCase):
+ """Tests for end position of AST nodes.
+
+ Testing end positions of nodes requires a bit of extra care
+ because of how LL parsers work.
+ """
+ def _check_end_pos(self, ast_node, end_lineno, end_col_offset):
+ self.assertEqual(ast_node.end_lineno, end_lineno)
+ self.assertEqual(ast_node.end_col_offset, end_col_offset)
+
+ def _check_content(self, source, ast_node, content):
+ self.assertEqual(ast.get_source_segment(source, ast_node), content)
+
+ def _parse_value(self, s):
+ # Use duck-typing to support both single expression
+ # and a right hand side of an assignment statement.
+ return ast.parse(s).body[0].value
+
+ def test_lambda(self):
+ s = 'lambda x, *y: None'
+ lam = self._parse_value(s)
+ self._check_content(s, lam.body, 'None')
+ self._check_content(s, lam.args.args[0], 'x')
+ self._check_content(s, lam.args.vararg, 'y')
+
+ def test_func_def(self):
+ s = dedent('''
+ def func(x: int,
+ *args: str,
+ z: float = 0,
+ **kwargs: Any) -> bool:
+ return True
+ ''').strip()
+ fdef = ast.parse(s).body[0]
+ self._check_end_pos(fdef, 5, 15)
+ self._check_content(s, fdef.body[0], 'return True')
+ self._check_content(s, fdef.args.args[0], 'x: int')
+ self._check_content(s, fdef.args.args[0].annotation, 'int')
+ self._check_content(s, fdef.args.kwarg, 'kwargs: Any')
+ self._check_content(s, fdef.args.kwarg.annotation, 'Any')
+
+ def test_call(self):
+ s = 'func(x, y=2, **kw)'
+ call = self._parse_value(s)
+ self._check_content(s, call.func, 'func')
+ self._check_content(s, call.keywords[0].value, '2')
+ self._check_content(s, call.keywords[1].value, 'kw')
+
+ def test_call_noargs(self):
+ s = 'x[0]()'
+ call = self._parse_value(s)
+ self._check_content(s, call.func, 'x[0]')
+ self._check_end_pos(call, 1, 6)
+
+ def test_class_def(self):
+ s = dedent('''
+ class C(A, B):
+ x: int = 0
+ ''').strip()
+ cdef = ast.parse(s).body[0]
+ self._check_end_pos(cdef, 2, 14)
+ self._check_content(s, cdef.bases[1], 'B')
+ self._check_content(s, cdef.body[0], 'x: int = 0')
+
+ def test_class_kw(self):
+ s = 'class S(metaclass=abc.ABCMeta): pass'
+ cdef = ast.parse(s).body[0]
+ self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta')
+
+ def test_multi_line_str(self):
+ s = dedent('''
+ x = """Some multi-line text.
+
+ It goes on starting from same indent."""
+ ''').strip()
+ assign = ast.parse(s).body[0]
+ self._check_end_pos(assign, 3, 40)
+ self._check_end_pos(assign.value, 3, 40)
+
+ def test_continued_str(self):
+ s = dedent('''
+ x = "first part" \\
+ "second part"
+ ''').strip()
+ assign = ast.parse(s).body[0]
+ self._check_end_pos(assign, 2, 13)
+ self._check_end_pos(assign.value, 2, 13)
+
+ def test_suites(self):
+ # We intentionally put these into the same string to check
+ # that empty lines are not part of the suite.
+ s = dedent('''
+ while True:
+ pass
+
+ if one():
+ x = None
+ elif other():
+ y = None
+ else:
+ z = None
+
+ for x, y in stuff:
+ assert True
+
+ try:
+ raise RuntimeError
+ except TypeError as e:
+ pass
+
+ pass
+ ''').strip()
+ mod = ast.parse(s)
+ while_loop = mod.body[0]
+ if_stmt = mod.body[1]
+ for_loop = mod.body[2]
+ try_stmt = mod.body[3]
+ pass_stmt = mod.body[4]
+
+ self._check_end_pos(while_loop, 2, 8)
+ self._check_end_pos(if_stmt, 9, 12)
+ self._check_end_pos(for_loop, 12, 15)
+ self._check_end_pos(try_stmt, 17, 8)
+ self._check_end_pos(pass_stmt, 19, 4)
+
+ self._check_content(s, while_loop.test, 'True')
+ self._check_content(s, if_stmt.body[0], 'x = None')
+ self._check_content(s, if_stmt.orelse[0].test, 'other()')
+ self._check_content(s, for_loop.target, 'x, y')
+ self._check_content(s, try_stmt.body[0], 'raise RuntimeError')
+ self._check_content(s, try_stmt.handlers[0].type, 'TypeError')
+
+ def test_fstring(self):
+ s = 'x = f"abc {x + y} abc"'
+ fstr = self._parse_value(s)
+ binop = fstr.values[1].value
+ self._check_content(s, binop, 'x + y')
+
+ def test_fstring_multi_line(self):
+ s = dedent('''
+ f"""Some multi-line text.
+ {
+ arg_one
+ +
+ arg_two
+ }
+ It goes on..."""
+ ''').strip()
+ fstr = self._parse_value(s)
+ binop = fstr.values[1].value
+ self._check_end_pos(binop, 5, 7)
+ self._check_content(s, binop.left, 'arg_one')
+ self._check_content(s, binop.right, 'arg_two')
+
+ def test_import_from_multi_line(self):
+ s = dedent('''
+ from x.y.z import (
+ a, b, c as c
+ )
+ ''').strip()
+ imp = ast.parse(s).body[0]
+ self._check_end_pos(imp, 3, 1)
+
+ def test_slices(self):
+ s1 = 'f()[1, 2] [0]'
+ s2 = 'x[ a.b: c.d]'
+ sm = dedent('''
+ x[ a.b: f () ,
+ g () : c.d
+ ]
+ ''').strip()
+ i1, i2, im = map(self._parse_value, (s1, s2, sm))
+ self._check_content(s1, i1.value, 'f()[1, 2]')
+ self._check_content(s1, i1.value.slice.value, '1, 2')
+ self._check_content(s2, i2.slice.lower, 'a.b')
+ self._check_content(s2, i2.slice.upper, 'c.d')
+ self._check_content(sm, im.slice.dims[0].upper, 'f ()')
+ self._check_content(sm, im.slice.dims[1].lower, 'g ()')
+ self._check_end_pos(im, 3, 3)
+
+ def test_binop(self):
+ s = dedent('''
+ (1 * 2 + (3 ) +
+ 4
+ )
+ ''').strip()
+ binop = self._parse_value(s)
+ self._check_end_pos(binop, 2, 6)
+ self._check_content(s, binop.right, '4')
+ self._check_content(s, binop.left, '1 * 2 + (3 )')
+ self._check_content(s, binop.left.right, '3')
+
+ def test_boolop(self):
+ s = dedent('''
+ if (one_condition and
+ (other_condition or yet_another_one)):
+ pass
+ ''').strip()
+ bop = ast.parse(s).body[0].test
+ self._check_end_pos(bop, 2, 44)
+ self._check_content(s, bop.values[1],
+ 'other_condition or yet_another_one')
+
+ def test_tuples(self):
+ s1 = 'x = () ;'
+ s2 = 'x = 1 , ;'
+ s3 = 'x = (1 , 2 ) ;'
+ sm = dedent('''
+ x = (
+ a, b,
+ )
+ ''').strip()
+ t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm))
+ self._check_content(s1, t1, '()')
+ self._check_content(s2, t2, '1 ,')
+ self._check_content(s3, t3, '(1 , 2 )')
+ self._check_end_pos(tm, 3, 1)
+
+ def test_attribute_spaces(self):
+ s = 'func(x. y .z)'
+ call = self._parse_value(s)
+ self._check_content(s, call, s)
+ self._check_content(s, call.args[0], 'x. y .z')
+
+ def test_displays(self):
+ s1 = '[{}, {1, }, {1, 2,} ]'
+ s2 = '{a: b, f (): g () ,}'
+ c1 = self._parse_value(s1)
+ c2 = self._parse_value(s2)
+ self._check_content(s1, c1.elts[0], '{}')
+ self._check_content(s1, c1.elts[1], '{1, }')
+ self._check_content(s1, c1.elts[2], '{1, 2,}')
+ self._check_content(s2, c2.keys[1], 'f ()')
+ self._check_content(s2, c2.values[1], 'g ()')
+
+ def test_comprehensions(self):
+ s = dedent('''
+ x = [{x for x, y in stuff
+ if cond.x} for stuff in things]
+ ''').strip()
+ cmp = self._parse_value(s)
+ self._check_end_pos(cmp, 2, 37)
+ self._check_content(s, cmp.generators[0].iter, 'things')
+ self._check_content(s, cmp.elt.generators[0].iter, 'stuff')
+ self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x')
+ self._check_content(s, cmp.elt.generators[0].target, 'x, y')
+
+ def test_yield_await(self):
+ s = dedent('''
+ async def f():
+ yield x
+ await y
+ ''').strip()
+ fdef = ast.parse(s).body[0]
+ self._check_content(s, fdef.body[0].value, 'yield x')
+ self._check_content(s, fdef.body[1].value, 'await y')
+
+ def test_source_segment_multi(self):
+ s_orig = dedent('''
+ x = (
+ a, b,
+ ) + ()
+ ''').strip()
+ s_tuple = dedent('''
+ (
+ a, b,
+ )
+ ''').strip()
+ binop = self._parse_value(s_orig)
+ self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple)
+
+ def test_source_segment_padded(self):
+ s_orig = dedent('''
+ class C:
+ def fun(self) -> None:
+ "ЖЖЖЖЖ"
+ ''').strip()
+ s_method = ' def fun(self) -> None:\n' \
+ ' "ЖЖЖЖЖ"'
+ cdef = ast.parse(s_orig).body[0]
+ self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True),
+ s_method)
+
+ def test_source_segment_endings(self):
+ s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n'
+ v, w, x, y, z = ast.parse(s).body
+ self._check_content(s, v, 'v = 1')
+ self._check_content(s, w, 'w = 1')
+ self._check_content(s, x, 'x = 1')
+ self._check_content(s, y, 'y = 1')
+ self._check_content(s, z, 'z = 1')
+
+ def test_source_segment_tabs(self):
+ s = dedent('''
+ class C:
+ \t\f def fun(self) -> None:
+ \t\f pass
+ ''').strip()
+ s_method = ' \t\f def fun(self) -> None:\n' \
+ ' \t\f pass'
+
+ cdef = ast.parse(s).body[0]
+ self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method)
+
+
def main():
if __name__ != '__main__':
return
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 274e260..9b58bb9 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -880,7 +880,7 @@ class STObjectTestCase(unittest.TestCase):
return 1 << (n - 1).bit_length()
basesize = support.calcobjsize('Pii')
- nodesize = struct.calcsize('hP3iP0h')
+ nodesize = struct.calcsize('hP3iP0h2i')
def sizeofchildren(node):
if node is None:
return 0
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst b/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst
new file mode 100644
index 0000000..2e618e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-01-19-19-41-53.bpo-33416.VDeOU5.rst
@@ -0,0 +1,2 @@
+Add end line and end column position information to the Python AST nodes.
+This is a C-level backwards incompatible change. \ No newline at end of file
diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
index 8f88657..eabc5c8 100644
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -920,7 +920,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_DECREF(elem);
return NULL;
}
- err = PyNode_AddChild(root, type, strn, *line_num, 0);
+ err = PyNode_AddChild(root, type, strn, *line_num, 0, *line_num, 0);
if (err == E_NOMEM) {
Py_DECREF(elem);
PyObject_FREE(strn);
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index eee982b..cedf37a 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -50,7 +50,7 @@ module Python
-- XXX Jython will be different
-- col_offset is the byte offset in the utf8 string the parser uses
- attributes (int lineno, int col_offset)
+ attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
-- BoolOp() can use left & right?
expr = BoolOp(boolop op, expr* values)
@@ -85,7 +85,7 @@ module Python
| Tuple(expr* elts, expr_context ctx)
-- col_offset is the byte offset in the utf8 string the parser uses
- attributes (int lineno, int col_offset)
+ attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
expr_context = Load | Store | Del | AugLoad | AugStore | Param
@@ -105,13 +105,13 @@ module Python
comprehension = (expr target, expr iter, expr* ifs, int is_async)
excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
- attributes (int lineno, int col_offset)
+ attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
arg? kwarg, expr* defaults)
arg = (identifier arg, expr? annotation)
- attributes (int lineno, int col_offset)
+ attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
-- keyword arguments supplied to call (NULL identifier for **kwargs)
keyword = (identifier? arg, expr value)
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 75fb78b..8640b29 100644
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1250,10 +1250,12 @@ def main(srcfile, dump_module=False):
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
c = ChainOfVisitors(TypeDefVisitor(f),
- StructVisitor(f),
- PrototypeVisitor(f),
- )
+ StructVisitor(f))
+
c.visit(mod)
+ f.write("// Note: these macros affect function definitions, not only call sites.\n")
+ PrototypeVisitor(f).visit(mod)
+ f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
diff --git a/Parser/node.c b/Parser/node.c
index 240d290..f1b70e0 100644
--- a/Parser/node.c
+++ b/Parser/node.c
@@ -13,6 +13,8 @@ PyNode_New(int type)
n->n_type = type;
n->n_str = NULL;
n->n_lineno = 0;
+ n->n_end_lineno = 0;
+ n->n_end_col_offset = -1;
n->n_nchildren = 0;
n->n_child = NULL;
return n;
@@ -75,14 +77,34 @@ fancy_roundup(int n)
fancy_roundup(n))
+void
+_PyNode_FinalizeEndPos(node *n)
+{
+ int nch = NCH(n);
+ node *last;
+ if (nch == 0) {
+ return;
+ }
+ last = CHILD(n, nch - 1);
+ _PyNode_FinalizeEndPos(last);
+ n->n_end_lineno = last->n_end_lineno;
+ n->n_end_col_offset = last->n_end_col_offset;
+}
+
int
-PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset)
+PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset,
+ int end_lineno, int end_col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
int required_capacity;
node *n;
+ // finalize end position of previous node (if any)
+ if (nch > 0) {
+ _PyNode_FinalizeEndPos(CHILD(n1, nch - 1));
+ }
+
if (nch == INT_MAX || nch < 0)
return E_OVERFLOW;
@@ -107,6 +129,8 @@ PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset)
n->n_str = str;
n->n_lineno = lineno;
n->n_col_offset = col_offset;
+ n->n_end_lineno = end_lineno; // this and below will be updates after all children are added.
+ n->n_end_col_offset = end_col_offset;
n->n_nchildren = 0;
n->n_child = NULL;
return 0;
diff --git a/Parser/parser.c b/Parser/parser.c
index 41072c4..a9916d3 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -105,11 +105,13 @@ PyParser_Delete(parser_state *ps)
/* PARSER STACK OPERATIONS */
static int
-shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset)
+shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset,
+ int end_lineno, int end_col_offset)
{
int err;
assert(!s_empty(s));
- err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset);
+ err = PyNode_AddChild(s->s_top->s_parent, type, str, lineno, col_offset,
+ end_lineno, end_col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
@@ -117,13 +119,15 @@ shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset)
}
static int
-push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
+push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset,
+ int end_lineno, int end_col_offset)
{
int err;
node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
- err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
+ err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset,
+ end_lineno, end_col_offset);
if (err)
return err;
s->s_top->s_state = newstate;
@@ -225,7 +229,9 @@ future_hack(parser_state *ps)
int
PyParser_AddToken(parser_state *ps, int type, char *str,
- int lineno, int col_offset, int *expected_ret)
+ int lineno, int col_offset,
+ int end_lineno, int end_col_offset,
+ int *expected_ret)
{
int ilabel;
int err;
@@ -257,7 +263,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str,
dfa *d1 = PyGrammar_FindDFA(
ps->p_grammar, nt);
if ((err = push(&ps->p_stack, nt, d1,
- arrow, lineno, col_offset)) > 0) {
+ arrow, lineno, col_offset,
+ end_lineno, end_col_offset)) > 0) {
D(printf(" MemError: push\n"));
return err;
}
@@ -267,7 +274,8 @@ PyParser_AddToken(parser_state *ps, int type, char *str,
/* Shift the token */
if ((err = shift(&ps->p_stack, type, str,
- x, lineno, col_offset)) > 0) {
+ x, lineno, col_offset,
+ end_lineno, end_col_offset)) > 0) {
D(printf(" MemError: shift.\n"));
return err;
}
diff --git a/Parser/parser.h b/Parser/parser.h
index 39df948..95cd39d 100644
--- a/Parser/parser.h
+++ b/Parser/parser.h
@@ -32,7 +32,9 @@ typedef struct {
parser_state *PyParser_New(grammar *g, int start);
void PyParser_Delete(parser_state *ps);
-int PyParser_AddToken(parser_state *ps, int type, char *str, int lineno, int col_offset,
+int PyParser_AddToken(parser_state *ps, int type, char *str,
+ int lineno, int col_offset,
+ int end_lineno, int end_col_offset,
int *expected_ret);
void PyGrammar_AddAccelerators(grammar *g);
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index d37e28a..2b5254a 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -187,7 +187,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
parser_state *ps;
node *n;
int started = 0;
- int col_offset;
+ int col_offset, end_col_offset;
if ((ps = PyParser_New(g, start)) == NULL) {
err_ret->error = E_NOMEM;
@@ -270,9 +270,16 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
col_offset = -1;
}
+ if (b != NULL && b >= tok->line_start) {
+ end_col_offset = Py_SAFE_DOWNCAST(b - tok->line_start,
+ intptr_t, int);
+ }
+ else {
+ end_col_offset = -1;
+ }
if ((err_ret->error =
PyParser_AddToken(ps, (int)type, str,
- lineno, col_offset,
+ lineno, col_offset, tok->lineno, end_col_offset,
&(err_ret->expected))) != E_OK) {
if (err_ret->error != E_DONE) {
PyObject_FREE(str);
@@ -368,6 +375,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
done:
PyTokenizer_Free(tok);
+ if (n != NULL) {
+ _PyNode_FinalizeEndPos(n);
+ }
return n;
}
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index bbe8e69..e6c5bfe 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -28,9 +28,13 @@ static char *Suite_fields[]={
static PyTypeObject *stmt_type;
_Py_IDENTIFIER(lineno);
_Py_IDENTIFIER(col_offset);
+_Py_IDENTIFIER(end_lineno);
+_Py_IDENTIFIER(end_col_offset);
static char *stmt_attributes[] = {
"lineno",
"col_offset",
+ "end_lineno",
+ "end_col_offset",
};
static PyObject* ast2obj_stmt(void*);
static PyTypeObject *FunctionDef_type;
@@ -189,6 +193,8 @@ static PyTypeObject *expr_type;
static char *expr_attributes[] = {
"lineno",
"col_offset",
+ "end_lineno",
+ "end_col_offset",
};
static PyObject* ast2obj_expr(void*);
static PyTypeObject *BoolOp_type;
@@ -427,6 +433,8 @@ static PyTypeObject *excepthandler_type;
static char *excepthandler_attributes[] = {
"lineno",
"col_offset",
+ "end_lineno",
+ "end_col_offset",
};
static PyObject* ast2obj_excepthandler(void*);
static PyTypeObject *ExceptHandler_type;
@@ -456,6 +464,8 @@ static PyObject* ast2obj_arg(void*);
static char *arg_attributes[] = {
"lineno",
"col_offset",
+ "end_lineno",
+ "end_col_offset",
};
_Py_IDENTIFIER(arg);
static char *arg_fields[]={
@@ -804,7 +814,7 @@ static int init_types(void)
if (!Suite_type) return 0;
stmt_type = make_type("stmt", &AST_type, NULL, 0);
if (!stmt_type) return 0;
- if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
+ if (!add_attributes(stmt_type, stmt_attributes, 4)) return 0;
FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
5);
if (!FunctionDef_type) return 0;
@@ -859,7 +869,7 @@ static int init_types(void)
if (!Continue_type) return 0;
expr_type = make_type("expr", &AST_type, NULL, 0);
if (!expr_type) return 0;
- if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
+ if (!add_attributes(expr_type, expr_attributes, 4)) return 0;
BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
if (!BoolOp_type) return 0;
BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
@@ -1082,7 +1092,7 @@ static int init_types(void)
if (!add_attributes(comprehension_type, NULL, 0)) return 0;
excepthandler_type = make_type("excepthandler", &AST_type, NULL, 0);
if (!excepthandler_type) return 0;
- if (!add_attributes(excepthandler_type, excepthandler_attributes, 2))
+ if (!add_attributes(excepthandler_type, excepthandler_attributes, 4))
return 0;
ExceptHandler_type = make_type("ExceptHandler", excepthandler_type,
ExceptHandler_fields, 3);
@@ -1092,7 +1102,7 @@ static int init_types(void)
if (!add_attributes(arguments_type, NULL, 0)) return 0;
arg_type = make_type("arg", &AST_type, arg_fields, 2);
if (!arg_type) return 0;
- if (!add_attributes(arg_type, arg_attributes, 2)) return 0;
+ if (!add_attributes(arg_type, arg_attributes, 4)) return 0;
keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
if (!keyword_type) return 0;
if (!add_attributes(keyword_type, NULL, 0)) return 0;
@@ -1181,8 +1191,8 @@ Suite(asdl_seq * body, PyArena *arena)
stmt_ty
FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
- decorator_list, expr_ty returns, int lineno, int col_offset,
- PyArena *arena)
+ decorator_list, expr_ty returns, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@@ -1206,13 +1216,15 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
p->v.FunctionDef.returns = returns;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
* decorator_list, expr_ty returns, int lineno, int col_offset,
- PyArena *arena)
+ int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@@ -1236,13 +1248,15 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
p->v.AsyncFunctionDef.returns = returns;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
- body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
- *arena)
+ body, asdl_seq * decorator_list, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!name) {
@@ -1261,11 +1275,14 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
p->v.ClassDef.decorator_list = decorator_list;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Return(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1275,11 +1292,14 @@ Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
p->v.Return.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
+Delete(asdl_seq * targets, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1289,12 +1309,14 @@ Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
p->v.Delete.targets = targets;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
- *arena)
+Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!value) {
@@ -1310,12 +1332,14 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
p->v.Assign.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
@@ -1342,12 +1366,15 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
p->v.AugAssign.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int
- lineno, int col_offset, PyArena *arena)
+ lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena)
{
stmt_ty p;
if (!target) {
@@ -1370,12 +1397,14 @@ AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int
p->v.AnnAssign.simple = simple;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
- lineno, int col_offset, PyArena *arena)
+ lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!target) {
@@ -1398,12 +1427,15 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
p->v.For.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
- lineno, int col_offset, PyArena *arena)
+ lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena)
{
stmt_ty p;
if (!target) {
@@ -1426,12 +1458,14 @@ AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
p->v.AsyncFor.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@@ -1448,12 +1482,14 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
p->v.While.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@@ -1470,12 +1506,14 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
p->v.If.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
- *arena)
+With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1486,12 +1524,14 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
p->v.With.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
- PyArena *arena)
+AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1502,11 +1542,14 @@ AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
p->v.AsyncWith.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
+Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1517,12 +1560,15 @@ Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
p->v.Raise.cause = cause;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq *
- finalbody, int lineno, int col_offset, PyArena *arena)
+ finalbody, int lineno, int col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1535,11 +1581,14 @@ Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq *
p->v.Try.finalbody = finalbody;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
+Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno,
+ int end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!test) {
@@ -1555,11 +1604,14 @@ Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
p->v.Assert.msg = msg;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Import(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1569,12 +1621,14 @@ Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
p->v.Import.names = names;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1586,11 +1640,14 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
p->v.ImportFrom.level = level;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Global(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1600,11 +1657,14 @@ Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
p->v.Global.names = names;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Nonlocal(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1614,11 +1674,14 @@ Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
p->v.Nonlocal.names = names;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
stmt_ty p;
if (!value) {
@@ -1633,11 +1696,14 @@ Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
p->v.Expr.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Pass(int lineno, int col_offset, PyArena *arena)
+Pass(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1646,11 +1712,14 @@ Pass(int lineno, int col_offset, PyArena *arena)
p->kind = Pass_kind;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Break(int lineno, int col_offset, PyArena *arena)
+Break(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1659,11 +1728,14 @@ Break(int lineno, int col_offset, PyArena *arena)
p->kind = Break_kind;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
stmt_ty
-Continue(int lineno, int col_offset, PyArena *arena)
+Continue(int lineno, int col_offset, int end_lineno, int end_col_offset,
+ PyArena *arena)
{
stmt_ty p;
p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1672,12 +1744,14 @@ Continue(int lineno, int col_offset, PyArena *arena)
p->kind = Continue_kind;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
- *arena)
+BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!op) {
@@ -1693,12 +1767,14 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
p->v.BoolOp.values = values;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
- PyArena *arena)
+ int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!left) {
@@ -1725,12 +1801,14 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
p->v.BinOp.right = right;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
- *arena)
+UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!op) {
@@ -1751,12 +1829,14 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
p->v.UnaryOp.operand = operand;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
- *arena)
+Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!args) {
@@ -1777,12 +1857,14 @@ Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
p->v.Lambda.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
- PyArena *arena)
+ int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!test) {
@@ -1809,12 +1891,14 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
p->v.IfExp.orelse = orelse;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
- *arena)
+Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1825,11 +1909,14 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
p->v.Dict.values = values;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena)
+Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1839,12 +1926,14 @@ Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena)
p->v.Set.elts = elts;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
- PyArena *arena)
+ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!elt) {
@@ -1860,12 +1949,14 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
p->v.ListComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
- *arena)
+SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!elt) {
@@ -1881,12 +1972,14 @@ SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
p->v.SetComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!key) {
@@ -1908,12 +2001,14 @@ DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int
p->v.DictComp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
- PyArena *arena)
+ int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!elt) {
@@ -1929,11 +2024,14 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
p->v.GeneratorExp.generators = generators;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Await(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -1948,11 +2046,14 @@ Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
p->v.Await.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Yield(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1962,11 +2063,14 @@ Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
p->v.Yield.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
+YieldFrom(expr_ty value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -1981,12 +2085,14 @@ YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
p->v.YieldFrom.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
- int col_offset, PyArena *arena)
+ int col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!left) {
@@ -2003,12 +2109,14 @@ Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
p->v.Compare.comparators = comparators;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!func) {
@@ -2025,12 +2133,15 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
p->v.Call.keywords = keywords;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
- int col_offset, PyArena *arena)
+ int col_offset, int end_lineno, int end_col_offset, PyArena
+ *arena)
{
expr_ty p;
if (!value) {
@@ -2047,11 +2158,14 @@ FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
p->v.FormattedValue.format_spec = format_spec;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
+JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2061,11 +2175,14 @@ JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
p->v.JoinedStr.values = values;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Constant(constant value, int lineno, int col_offset, PyArena *arena)
+Constant(constant value, int lineno, int col_offset, int end_lineno, int
+ end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -2080,12 +2197,14 @@ Constant(constant value, int lineno, int col_offset, PyArena *arena)
p->v.Constant.value = value;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -2112,12 +2231,14 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
p->v.Attribute.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -2144,12 +2265,14 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
p->v.Subscript.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena
- *arena)
+Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!value) {
@@ -2170,12 +2293,14 @@ Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena
p->v.Starred.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
- *arena)
+Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!id) {
@@ -2196,12 +2321,14 @@ Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
p->v.Name.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
- *arena)
+List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!ctx) {
@@ -2217,12 +2344,14 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
p->v.List.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
expr_ty
-Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
- *arena)
+Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
expr_ty p;
if (!ctx) {
@@ -2238,6 +2367,8 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
p->v.Tuple.ctx = ctx;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
@@ -2311,7 +2442,7 @@ comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async,
excepthandler_ty
ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int
- col_offset, PyArena *arena)
+ col_offset, int end_lineno, int end_col_offset, PyArena *arena)
{
excepthandler_ty p;
p = (excepthandler_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2323,6 +2454,8 @@ ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int
p->v.ExceptHandler.body = body;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
@@ -2344,8 +2477,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq *
}
arg_ty
-arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
- *arena)
+arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int
+ end_lineno, int end_col_offset, PyArena *arena)
{
arg_ty p;
if (!arg) {
@@ -2360,6 +2493,8 @@ arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
p->annotation = annotation;
p->lineno = lineno;
p->col_offset = col_offset;
+ p->end_lineno = end_lineno;
+ p->end_col_offset = end_col_offset;
return p;
}
@@ -2886,6 +3021,16 @@ ast2obj_stmt(void* _o)
if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
goto failed;
Py_DECREF(value);
+ value = ast2obj_int(o->end_lineno);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_int(o->end_col_offset);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+ goto failed;
+ Py_DECREF(value);
return result;
failed:
Py_XDECREF(value);
@@ -3281,6 +3426,16 @@ ast2obj_expr(void* _o)
if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
goto failed;
Py_DECREF(value);
+ value = ast2obj_int(o->end_lineno);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_int(o->end_col_offset);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+ goto failed;
+ Py_DECREF(value);
return result;
failed:
Py_XDECREF(value);
@@ -3571,6 +3726,16 @@ ast2obj_excepthandler(void* _o)
if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
goto failed;
Py_DECREF(value);
+ value = ast2obj_int(o->end_lineno);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_int(o->end_col_offset);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+ goto failed;
+ Py_DECREF(value);
return result;
failed:
Py_XDECREF(value);
@@ -3657,6 +3822,16 @@ ast2obj_arg(void* _o)
if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
goto failed;
Py_DECREF(value);
+ value = ast2obj_int(o->end_lineno);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_int(o->end_col_offset);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+ goto failed;
+ Py_DECREF(value);
return result;
failed:
Py_XDECREF(value);
@@ -3922,6 +4097,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
PyObject *tmp = NULL;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
if (obj == Py_None) {
*out = NULL;
@@ -3953,6 +4130,32 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
+ if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_lineno = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_lineno, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_col_offset = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_col_offset, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionDef_type);
if (isinstance == -1) {
return 1;
@@ -4064,7 +4267,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = FunctionDef(name, args, body, decorator_list, returns, lineno,
- col_offset, arena);
+ col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4179,7 +4382,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = AsyncFunctionDef(name, args, body, decorator_list, returns,
- lineno, col_offset, arena);
+ lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4328,7 +4532,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = ClassDef(name, bases, keywords, body, decorator_list, lineno,
- col_offset, arena);
+ col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4352,7 +4556,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Return(value, lineno, col_offset, arena);
+ *out = Return(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4393,7 +4598,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Delete(targets, lineno, col_offset, arena);
+ *out = Delete(targets, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4448,7 +4654,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Assign(targets, value, lineno, col_offset, arena);
+ *out = Assign(targets, value, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4500,7 +4707,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = AugAssign(target, op, value, lineno, col_offset, arena);
+ *out = AugAssign(target, op, value, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4567,7 +4775,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = AnnAssign(target, annotation, value, simple, lineno, col_offset,
- arena);
+ end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4667,7 +4875,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = For(target, iter, body, orelse, lineno, col_offset, arena);
+ *out = For(target, iter, body, orelse, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4767,7 +4976,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena);
+ *out = AsyncFor(target, iter, body, orelse, lineno, col_offset,
+ end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4853,7 +5063,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = While(test, body, orelse, lineno, col_offset, arena);
+ *out = While(test, body, orelse, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -4939,7 +5150,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = If(test, body, orelse, lineno, col_offset, arena);
+ *out = If(test, body, orelse, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5011,7 +5223,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = With(items, body, lineno, col_offset, arena);
+ *out = With(items, body, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5083,7 +5296,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = AsyncWith(items, body, lineno, col_offset, arena);
+ *out = AsyncWith(items, body, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5121,7 +5335,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Raise(exc, cause, lineno, col_offset, arena);
+ *out = Raise(exc, cause, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5256,7 +5471,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = Try(body, handlers, orelse, finalbody, lineno, col_offset,
- arena);
+ end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5294,7 +5509,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Assert(test, msg, lineno, col_offset, arena);
+ *out = Assert(test, msg, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5335,7 +5551,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Import(names, lineno, col_offset, arena);
+ *out = Import(names, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5404,7 +5621,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = ImportFrom(module, names, level, lineno, col_offset, arena);
+ *out = ImportFrom(module, names, level, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5445,7 +5663,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Global(names, lineno, col_offset, arena);
+ *out = Global(names, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5486,7 +5705,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Nonlocal(names, lineno, col_offset, arena);
+ *out = Nonlocal(names, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5510,7 +5730,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Expr(value, lineno, col_offset, arena);
+ *out = Expr(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5520,7 +5741,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
if (isinstance) {
- *out = Pass(lineno, col_offset, arena);
+ *out = Pass(lineno, col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5530,7 +5751,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
if (isinstance) {
- *out = Break(lineno, col_offset, arena);
+ *out = Break(lineno, col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5540,7 +5761,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
}
if (isinstance) {
- *out = Continue(lineno, col_offset, arena);
+ *out = Continue(lineno, col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5559,6 +5780,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
PyObject *tmp = NULL;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
if (obj == Py_None) {
*out = NULL;
@@ -5590,6 +5813,32 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
+ if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_lineno = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_lineno, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_col_offset = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_col_offset, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)BoolOp_type);
if (isinstance == -1) {
return 1;
@@ -5641,7 +5890,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = BoolOp(op, values, lineno, col_offset, arena);
+ *out = BoolOp(op, values, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5693,7 +5943,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = BinOp(left, op, right, lineno, col_offset, arena);
+ *out = BinOp(left, op, right, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5731,7 +5982,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = UnaryOp(op, operand, lineno, col_offset, arena);
+ *out = UnaryOp(op, operand, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5769,7 +6021,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Lambda(args, body, lineno, col_offset, arena);
+ *out = Lambda(args, body, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5821,7 +6074,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = IfExp(test, body, orelse, lineno, col_offset, arena);
+ *out = IfExp(test, body, orelse, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5893,7 +6147,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Dict(keys, values, lineno, col_offset, arena);
+ *out = Dict(keys, values, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5934,7 +6189,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Set(elts, lineno, col_offset, arena);
+ *out = Set(elts, lineno, col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -5989,7 +6244,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = ListComp(elt, generators, lineno, col_offset, arena);
+ *out = ListComp(elt, generators, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6044,7 +6300,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = SetComp(elt, generators, lineno, col_offset, arena);
+ *out = SetComp(elt, generators, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6113,7 +6370,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = DictComp(key, value, generators, lineno, col_offset, arena);
+ *out = DictComp(key, value, generators, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6168,7 +6426,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = GeneratorExp(elt, generators, lineno, col_offset, arena);
+ *out = GeneratorExp(elt, generators, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6192,7 +6451,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Await(value, lineno, col_offset, arena);
+ *out = Await(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6216,7 +6476,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Yield(value, lineno, col_offset, arena);
+ *out = Yield(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6240,7 +6501,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = YieldFrom(value, lineno, col_offset, arena);
+ *out = YieldFrom(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6326,7 +6588,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Compare(left, ops, comparators, lineno, col_offset, arena);
+ *out = Compare(left, ops, comparators, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6412,7 +6675,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = Call(func, args, keywords, lineno, col_offset, arena);
+ *out = Call(func, args, keywords, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6465,7 +6729,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
Py_CLEAR(tmp);
}
*out = FormattedValue(value, conversion, format_spec, lineno,
- col_offset, arena);
+ col_offset, end_lineno, end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6506,7 +6770,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = JoinedStr(values, lineno, col_offset, arena);
+ *out = JoinedStr(values, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6530,7 +6795,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Constant(value, lineno, col_offset, arena);
+ *out = Constant(value, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6582,7 +6848,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Attribute(value, attr, ctx, lineno, col_offset, arena);
+ *out = Attribute(value, attr, ctx, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6634,7 +6901,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Subscript(value, slice, ctx, lineno, col_offset, arena);
+ *out = Subscript(value, slice, ctx, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6672,7 +6940,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Starred(value, ctx, lineno, col_offset, arena);
+ *out = Starred(value, ctx, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6710,7 +6979,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Name(id, ctx, lineno, col_offset, arena);
+ *out = Name(id, ctx, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6765,7 +7035,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = List(elts, ctx, lineno, col_offset, arena);
+ *out = List(elts, ctx, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -6820,7 +7091,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = Tuple(elts, ctx, lineno, col_offset, arena);
+ *out = Tuple(elts, ctx, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -7389,6 +7661,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
PyObject *tmp = NULL;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
if (obj == Py_None) {
*out = NULL;
@@ -7420,6 +7694,32 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
+ if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_lineno = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_lineno, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_col_offset = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_col_offset, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type);
if (isinstance == -1) {
return 1;
@@ -7485,7 +7785,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
}
Py_CLEAR(tmp);
}
- *out = ExceptHandler(type, name, body, lineno, col_offset, arena);
+ *out = ExceptHandler(type, name, body, lineno, col_offset, end_lineno,
+ end_col_offset, arena);
if (*out == NULL) goto failed;
return 0;
}
@@ -7669,6 +7970,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
expr_ty annotation;
int lineno;
int col_offset;
+ int end_lineno;
+ int end_col_offset;
if (_PyObject_LookupAttrId(obj, &PyId_arg, &tmp) < 0) {
return 1;
@@ -7722,7 +8025,34 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
if (res != 0) goto failed;
Py_CLEAR(tmp);
}
- *out = arg(arg, annotation, lineno, col_offset, arena);
+ if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_lineno = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_lineno, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+ return 1;
+ }
+ if (tmp == NULL || tmp == Py_None) {
+ Py_CLEAR(tmp);
+ end_col_offset = 0;
+ }
+ else {
+ int res;
+ res = obj2ast_int(tmp, &end_col_offset, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ }
+ *out = arg(arg, annotation, lineno, col_offset, end_lineno, end_col_offset,
+ arena);
return 0;
failed:
Py_XDECREF(tmp);
diff --git a/Python/ast.c b/Python/ast.c
index d71f44a..855acca 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -580,10 +580,11 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty,
- const node *);
+ const node *, const node *);
static PyObject *parsenumber(struct compiling *, const char *);
static expr_ty parsestrplus(struct compiling *, const node *n);
+static void get_last_end_pos(asdl_seq *, int *, int *);
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
@@ -810,6 +811,7 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
if (!stmts)
goto out;
asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
arena));
if (!asdl_seq_GET(stmts, 0))
goto out;
@@ -940,6 +942,8 @@ copy_location(expr_ty e, const node *n)
if (e) {
e->lineno = LINENO(n);
e->col_offset = n->n_col_offset;
+ e->end_lineno = n->n_end_lineno;
+ e->end_col_offset = n->n_end_col_offset;
}
return e;
}
@@ -1228,7 +1232,8 @@ ast_for_arg(struct compiling *c, const node *n)
return NULL;
}
- ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena);
+ ret = arg(name, annotation, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!ret)
return NULL;
return ret;
@@ -1287,6 +1292,7 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
if (forbidden_name(c, argname, ch, 0))
goto error;
arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset,
+ ch->n_end_lineno, ch->n_end_col_offset,
c->c_arena);
if (!arg)
goto error;
@@ -1480,16 +1486,19 @@ ast_for_dotted_name(struct compiling *c, const node *n)
identifier id;
int lineno, col_offset;
int i;
+ node *ch;
REQ(n, dotted_name);
lineno = LINENO(n);
col_offset = n->n_col_offset;
- id = NEW_IDENTIFIER(CHILD(n, 0));
+ ch = CHILD(n, 0);
+ id = NEW_IDENTIFIER(ch);
if (!id)
return NULL;
- e = Name(id, Load, lineno, col_offset, c->c_arena);
+ e = Name(id, Load, lineno, col_offset,
+ ch->n_end_lineno, ch->n_end_col_offset, c->c_arena);
if (!e)
return NULL;
@@ -1497,7 +1506,8 @@ ast_for_dotted_name(struct compiling *c, const node *n)
id = NEW_IDENTIFIER(CHILD(n, i));
if (!id)
return NULL;
- e = Attribute(e, id, Load, lineno, col_offset, c->c_arena);
+ e = Attribute(e, id, Load, lineno, col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!e)
return NULL;
}
@@ -1526,13 +1536,13 @@ ast_for_decorator(struct compiling *c, const node *n)
}
else if (NCH(n) == 5) { /* Call with no arguments */
d = Call(name_expr, NULL, NULL, LINENO(n),
- n->n_col_offset, c->c_arena);
+ n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!d)
return NULL;
name_expr = NULL;
}
else {
- d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2));
+ d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2), CHILD(n, 4));
if (!d)
return NULL;
name_expr = NULL;
@@ -1573,6 +1583,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
asdl_seq *body;
expr_ty returns = NULL;
int name_i = 1;
+ int end_lineno, end_col_offset;
REQ(n, funcdef);
@@ -1593,13 +1604,14 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0,
body = ast_for_suite(c, CHILD(n, name_i + 3));
if (!body)
return NULL;
+ get_last_end_pos(body, &end_lineno, &end_col_offset);
if (is_async)
return AsyncFunctionDef(name, args, body, decorator_seq, returns,
- LINENO(n0), n0->n_col_offset, c->c_arena);
+ LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena);
else
return FunctionDef(name, args, body, decorator_seq, returns,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena);
}
static stmt_ty
@@ -1704,7 +1716,8 @@ ast_for_lambdef(struct compiling *c, const node *n)
return NULL;
}
- return Lambda(args, expression, LINENO(n), n->n_col_offset, c->c_arena);
+ return Lambda(args, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
@@ -1724,6 +1737,7 @@ ast_for_ifexpr(struct compiling *c, const node *n)
if (!orelse)
return NULL;
return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
}
@@ -1852,8 +1866,9 @@ ast_for_comprehension(struct compiling *c, const node *n)
comp = comprehension(first, expression, NULL,
is_async, c->c_arena);
else
- comp = comprehension(Tuple(t, Store, first->lineno,
- first->col_offset, c->c_arena),
+ comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
+ for_ch->n_end_lineno, for_ch->n_end_col_offset,
+ c->c_arena),
expression, NULL, is_async, c->c_arena);
if (!comp)
return NULL;
@@ -1918,11 +1933,14 @@ ast_for_itercomp(struct compiling *c, const node *n, int type)
return NULL;
if (type == COMP_GENEXP)
- return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+ return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else if (type == COMP_LISTCOMP)
- return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+ return ListComp(elt, comps, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
- return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
+ return SetComp(elt, comps, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
@@ -1984,7 +2002,8 @@ ast_for_dictcomp(struct compiling *c, const node *n)
if (!comps)
return NULL;
- return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena);
+ return DictComp(key, value, comps, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
@@ -2017,7 +2036,8 @@ ast_for_dictdisplay(struct compiling *c, const node *n)
}
keys->size = j;
values->size = j;
- return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
+ return Dict(keys, values, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
@@ -2060,7 +2080,8 @@ ast_for_setdisplay(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(elts, i / 2, expression);
}
- return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
+ return Set(elts, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
@@ -2079,17 +2100,21 @@ ast_for_atom(struct compiling *c, const node *n)
size_t len = strlen(s);
if (len >= 4 && len <= 5) {
if (!strcmp(s, "None"))
- return Constant(Py_None, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(Py_None, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!strcmp(s, "True"))
- return Constant(Py_True, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(Py_True, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!strcmp(s, "False"))
- return Constant(Py_False, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(Py_False, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
name = new_identifier(s, c);
if (!name)
return NULL;
/* All names start in Load context, but may later be changed. */
- return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return Name(name, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case STRING: {
expr_ty str = parsestrplus(c, n);
@@ -2128,15 +2153,18 @@ ast_for_atom(struct compiling *c, const node *n)
Py_DECREF(pynum);
return NULL;
}
- return Constant(pynum, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(pynum, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case ELLIPSIS: /* Ellipsis */
- return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(Py_Ellipsis, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
if (TYPE(ch) == RPAR)
- return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return Tuple(NULL, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
@@ -2156,7 +2184,8 @@ ast_for_atom(struct compiling *c, const node *n)
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
- return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return List(NULL, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
@@ -2164,7 +2193,8 @@ ast_for_atom(struct compiling *c, const node *n)
if (!elts)
return NULL;
- return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return List(elts, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
return copy_location(ast_for_listcomp(c, ch), n);
@@ -2178,7 +2208,8 @@ ast_for_atom(struct compiling *c, const node *n)
ch = CHILD(n, 1);
if (TYPE(ch) == RBRACE) {
/* It's an empty dict. */
- return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ return Dict(NULL, NULL, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR);
@@ -2306,6 +2337,7 @@ ast_for_binop(struct compiling *c, const node *n)
return NULL;
result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
+ CHILD(n, 2)->n_end_lineno, CHILD(n, 2)->n_end_col_offset,
c->c_arena);
if (!result)
return NULL;
@@ -2325,6 +2357,8 @@ ast_for_binop(struct compiling *c, const node *n)
tmp_result = BinOp(result, newoperator, tmp,
LINENO(next_oper), next_oper->n_col_offset,
+ CHILD(n, i * 2 + 2)->n_end_lineno,
+ CHILD(n, i * 2 + 2)->n_end_col_offset,
c->c_arena);
if (!tmp_result)
return NULL;
@@ -2340,20 +2374,22 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
subscriptlist: subscript (',' subscript)* [',']
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
*/
+ const node *n_copy = n;
REQ(n, trailer);
if (TYPE(CHILD(n, 0)) == LPAR) {
if (NCH(n) == 2)
- return Call(left_expr, NULL, NULL, LINENO(n),
- n->n_col_offset, c->c_arena);
+ return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else
- return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0));
+ return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2));
}
else if (TYPE(CHILD(n, 0)) == DOT) {
PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
if (!attr_id)
return NULL;
return Attribute(left_expr, attr_id, Load,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
REQ(CHILD(n, 0), LSQB);
@@ -2364,6 +2400,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
if (!slc)
return NULL;
return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset,
+ n_copy->n_end_lineno, n_copy->n_end_col_offset,
c->c_arena);
}
else {
@@ -2389,7 +2426,8 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
}
if (!simple) {
return Subscript(left_expr, ExtSlice(slices, c->c_arena),
- Load, LINENO(n), n->n_col_offset, c->c_arena);
+ Load, LINENO(n), n->n_col_offset,
+ n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena);
}
/* extract Index values and put them in a Tuple */
elts = _Py_asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
@@ -2400,11 +2438,13 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
assert(slc->kind == Index_kind && slc->v.Index.value);
asdl_seq_SET(elts, j, slc->v.Index.value);
}
- e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ e = Tuple(elts, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!e)
return NULL;
return Subscript(left_expr, Index(e, c->c_arena),
- Load, LINENO(n), n->n_col_offset, c->c_arena);
+ Load, LINENO(n), n->n_col_offset,
+ n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena);
}
}
}
@@ -2421,13 +2461,16 @@ ast_for_factor(struct compiling *c, const node *n)
switch (TYPE(CHILD(n, 0))) {
case PLUS:
return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
case MINUS:
return UnaryOp(USub, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
case TILDE:
- return UnaryOp(Invert, expression, LINENO(n),
- n->n_col_offset, c->c_arena);
+ return UnaryOp(Invert, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
+ c->c_arena);
}
PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
TYPE(CHILD(n, 0)));
@@ -2454,7 +2497,8 @@ ast_for_atom_expr(struct compiling *c, const node *n)
if (nch == 1)
return e;
if (start && nch == 2) {
- return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+ return Await(e, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
for (i = start + 1; i < nch; i++) {
@@ -2471,7 +2515,8 @@ ast_for_atom_expr(struct compiling *c, const node *n)
if (start) {
/* there was an 'await' */
- return Await(e, LINENO(n), n->n_col_offset, c->c_arena);
+ return Await(e, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
return e;
@@ -2494,7 +2539,8 @@ ast_for_power(struct compiling *c, const node *n)
expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
if (!f)
return NULL;
- e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
+ e = BinOp(e, Pow, f, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
return e;
}
@@ -2510,7 +2556,8 @@ ast_for_starred(struct compiling *c, const node *n)
return NULL;
/* The Load context is changed later. */
- return Starred(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return Starred(tmp, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
@@ -2569,9 +2616,11 @@ ast_for_expr(struct compiling *c, const node *n)
}
if (!strcmp(STR(CHILD(n, 1)), "and"))
return BoolOp(And, seq, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
assert(!strcmp(STR(CHILD(n, 1)), "or"));
- return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
+ return BoolOp(Or, seq, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case not_test:
if (NCH(n) == 1) {
n = CHILD(n, 0);
@@ -2583,6 +2632,7 @@ ast_for_expr(struct compiling *c, const node *n)
return NULL;
return UnaryOp(Not, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
}
case comparison:
@@ -2622,8 +2672,8 @@ ast_for_expr(struct compiling *c, const node *n)
return NULL;
}
- return Compare(expression, ops, cmps, LINENO(n),
- n->n_col_offset, c->c_arena);
+ return Compare(expression, ops, cmps, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
break;
@@ -2663,8 +2713,10 @@ ast_for_expr(struct compiling *c, const node *n)
return NULL;
}
if (is_from)
- return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena);
- return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
+ return YieldFrom(exp, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
+ return Yield(exp, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case factor:
if (NCH(n) == 1) {
@@ -2684,7 +2736,7 @@ ast_for_expr(struct compiling *c, const node *n)
static expr_ty
ast_for_call(struct compiling *c, const node *n, expr_ty func,
- const node *maybegenbeg)
+ const node *maybegenbeg, const node *closepar)
{
/*
arglist: argument (',' argument)* [',']
@@ -2773,6 +2825,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func,
return NULL;
starred = Starred(e, Load, LINENO(chch),
chch->n_col_offset,
+ chch->n_end_lineno, chch->n_end_col_offset,
c->c_arena);
if (!starred)
return NULL;
@@ -2864,7 +2917,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func,
}
}
- return Call(func, args, keywords, func->lineno, func->col_offset, c->c_arena);
+ return Call(func, args, keywords, func->lineno, func->col_offset,
+ closepar->n_end_lineno, closepar->n_end_col_offset, c->c_arena);
}
static expr_ty
@@ -2887,7 +2941,8 @@ ast_for_testlist(struct compiling *c, const node* n)
asdl_seq *tmp = seq_for_testlist(c, n);
if (!tmp)
return NULL;
- return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
+ return Tuple(tmp, Load, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
}
@@ -2909,7 +2964,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (!e)
return NULL;
- return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
+ return Expr(e, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else if (TYPE(CHILD(n, 1)) == augassign) {
expr_ty expr1, expr2;
@@ -2947,7 +3003,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
if (!newoperator)
return NULL;
- return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
+ return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else if (TYPE(CHILD(n, 1)) == annassign) {
expr_ty expr1, expr2, expr3;
@@ -3007,7 +3064,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
}
if (NCH(ann) == 2) {
return AnnAssign(expr1, expr2, NULL, simple,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
ch = CHILD(ann, 3);
@@ -3016,7 +3074,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
return NULL;
}
return AnnAssign(expr1, expr2, expr3, simple,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
}
else {
@@ -3054,7 +3113,8 @@ ast_for_expr_stmt(struct compiling *c, const node *n)
expression = ast_for_expr(c, value);
if (!expression)
return NULL;
- return Assign(targets, expression, LINENO(n), n->n_col_offset, c->c_arena);
+ return Assign(targets, expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
}
@@ -3093,7 +3153,8 @@ ast_for_del_stmt(struct compiling *c, const node *n)
expr_list = ast_for_exprlist(c, CHILD(n, 1), Del);
if (!expr_list)
return NULL;
- return Delete(expr_list, LINENO(n), n->n_col_offset, c->c_arena);
+ return Delete(expr_list, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static stmt_ty
@@ -3115,27 +3176,33 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
ch = CHILD(n, 0);
switch (TYPE(ch)) {
case break_stmt:
- return Break(LINENO(n), n->n_col_offset, c->c_arena);
+ return Break(LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case continue_stmt:
- return Continue(LINENO(n), n->n_col_offset, c->c_arena);
+ return Continue(LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case yield_stmt: { /* will reduce to yield_expr */
expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
if (!exp)
return NULL;
- return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
+ return Expr(exp, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case return_stmt:
if (NCH(ch) == 1)
- return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ return Return(NULL, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else {
expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
if (!expression)
return NULL;
- return Return(expression, LINENO(n), n->n_col_offset, c->c_arena);
+ return Return(expression, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case raise_stmt:
if (NCH(ch) == 1)
- return Raise(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ return Raise(NULL, NULL, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else if (NCH(ch) >= 2) {
expr_ty cause = NULL;
expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
@@ -3146,7 +3213,8 @@ ast_for_flow_stmt(struct compiling *c, const node *n)
if (!cause)
return NULL;
}
- return Raise(expression, cause, LINENO(n), n->n_col_offset, c->c_arena);
+ return Raise(expression, cause, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
/* fall through */
default:
@@ -3307,11 +3375,14 @@ ast_for_import_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(aliases, i / 2, import_alias);
}
- return Import(aliases, lineno, col_offset, c->c_arena);
+ // Even though n is modified above, the end position is not changed
+ return Import(aliases, lineno, col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else if (TYPE(n) == import_from) {
int n_children;
int idx, ndots = 0;
+ const node *n_copy = n;
alias_ty mod = NULL;
identifier modname = NULL;
@@ -3382,6 +3453,7 @@ ast_for_import_stmt(struct compiling *c, const node *n)
if (mod != NULL)
modname = mod->name;
return ImportFrom(modname, aliases, ndots, lineno, col_offset,
+ n_copy->n_end_lineno, n_copy->n_end_col_offset,
c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@@ -3408,7 +3480,8 @@ ast_for_global_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(s, i / 2, name);
}
- return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
+ return Global(s, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static stmt_ty
@@ -3429,7 +3502,8 @@ ast_for_nonlocal_stmt(struct compiling *c, const node *n)
return NULL;
asdl_seq_SET(s, i / 2, name);
}
- return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena);
+ return Nonlocal(s, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static stmt_ty
@@ -3441,7 +3515,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
expr_ty expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
- return Assert(expression, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ return Assert(expression, NULL, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else if (NCH(n) == 4) {
expr_ty expr1, expr2;
@@ -3453,7 +3528,8 @@ ast_for_assert_stmt(struct compiling *c, const node *n)
if (!expr2)
return NULL;
- return Assert(expr1, expr2, LINENO(n), n->n_col_offset, c->c_arena);
+ return Assert(expr1, expr2, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
"improper number of parts to 'assert' statement: %d",
@@ -3527,6 +3603,20 @@ ast_for_suite(struct compiling *c, const node *n)
return seq;
}
+static void
+get_last_end_pos(asdl_seq *s, int *end_lineno, int *end_col_offset)
+{
+ int tot = asdl_seq_LEN(s);
+ // Suite should not be empty, but it is safe to just ignore it
+ // if it will ever occur.
+ if (!tot) {
+ return;
+ }
+ stmt_ty last = asdl_seq_GET(s, tot - 1);
+ *end_lineno = last->end_lineno;
+ *end_col_offset = last->end_col_offset;
+}
+
static stmt_ty
ast_for_if_stmt(struct compiling *c, const node *n)
{
@@ -3534,6 +3624,7 @@ ast_for_if_stmt(struct compiling *c, const node *n)
['else' ':' suite]
*/
char *s;
+ int end_lineno, end_col_offset;
REQ(n, if_stmt);
@@ -3547,9 +3638,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
- c->c_arena);
+ end_lineno, end_col_offset, c->c_arena);
}
s = STR(CHILD(n, 4));
@@ -3570,9 +3662,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
seq2 = ast_for_suite(c, CHILD(n, 6));
if (!seq2)
return NULL;
+ get_last_end_pos(seq2, &end_lineno, &end_col_offset);
return If(expression, seq1, seq2, LINENO(n), n->n_col_offset,
- c->c_arena);
+ end_lineno, end_col_offset, c->c_arena);
}
else if (s[2] == 'i') {
int i, n_elif, has_else = 0;
@@ -3604,12 +3697,13 @@ ast_for_if_stmt(struct compiling *c, const node *n)
suite_seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1));
if (!suite_seq2)
return NULL;
+ get_last_end_pos(suite_seq2, &end_lineno, &end_col_offset);
asdl_seq_SET(orelse, 0,
If(expression, suite_seq, suite_seq2,
LINENO(CHILD(n, NCH(n) - 6)),
CHILD(n, NCH(n) - 6)->n_col_offset,
- c->c_arena));
+ end_lineno, end_col_offset, c->c_arena));
/* the just-created orelse handled the last elif */
n_elif--;
}
@@ -3626,10 +3720,16 @@ ast_for_if_stmt(struct compiling *c, const node *n)
if (!suite_seq)
return NULL;
+ if (orelse != NULL) {
+ get_last_end_pos(orelse, &end_lineno, &end_col_offset);
+ } else {
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+ }
asdl_seq_SET(newobj, 0,
If(expression, suite_seq, orelse,
LINENO(CHILD(n, off)),
- CHILD(n, off)->n_col_offset, c->c_arena));
+ CHILD(n, off)->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena));
orelse = newobj;
}
expression = ast_for_expr(c, CHILD(n, 1));
@@ -3638,8 +3738,10 @@ ast_for_if_stmt(struct compiling *c, const node *n)
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
+ get_last_end_pos(orelse, &end_lineno, &end_col_offset);
return If(expression, suite_seq, orelse,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@@ -3652,6 +3754,7 @@ ast_for_while_stmt(struct compiling *c, const node *n)
{
/* while_stmt: 'while' test ':' suite ['else' ':' suite] */
REQ(n, while_stmt);
+ int end_lineno, end_col_offset;
if (NCH(n) == 4) {
expr_ty expression;
@@ -3663,7 +3766,9 @@ ast_for_while_stmt(struct compiling *c, const node *n)
suite_seq = ast_for_suite(c, CHILD(n, 3));
if (!suite_seq)
return NULL;
- return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset, c->c_arena);
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+ return While(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
else if (NCH(n) == 7) {
expr_ty expression;
@@ -3678,8 +3783,10 @@ ast_for_while_stmt(struct compiling *c, const node *n)
seq2 = ast_for_suite(c, CHILD(n, 6));
if (!seq2)
return NULL;
+ get_last_end_pos(seq2, &end_lineno, &end_col_offset);
- return While(expression, seq1, seq2, LINENO(n), n->n_col_offset, c->c_arena);
+ return While(expression, seq1, seq2, LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@@ -3696,6 +3803,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
expr_ty expression;
expr_ty target, first;
const node *node_target;
+ int end_lineno, end_col_offset;
/* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */
REQ(n, for_stmt);
@@ -3715,7 +3823,9 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
if (NCH(node_target) == 1)
target = first;
else
- target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena);
+ target = Tuple(_target, Store, first->lineno, first->col_offset,
+ node_target->n_end_lineno, node_target->n_end_col_offset,
+ c->c_arena);
expression = ast_for_testlist(c, CHILD(n, 3));
if (!expression)
@@ -3724,20 +3834,26 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
if (!suite_seq)
return NULL;
+ if (seq != NULL) {
+ get_last_end_pos(seq, &end_lineno, &end_col_offset);
+ } else {
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
+ }
if (is_async)
return AsyncFor(target, expression, suite_seq, seq,
LINENO(n0), n0->n_col_offset,
- c->c_arena);
+ end_lineno, end_col_offset, c->c_arena);
else
return For(target, expression, suite_seq, seq,
LINENO(n), n->n_col_offset,
- c->c_arena);
+ end_lineno, end_col_offset, c->c_arena);
}
static excepthandler_ty
ast_for_except_clause(struct compiling *c, const node *exc, node *body)
{
/* except_clause: 'except' [test ['as' test]] */
+ int end_lineno, end_col_offset;
REQ(exc, except_clause);
REQ(body, suite);
@@ -3745,9 +3861,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
asdl_seq *suite_seq = ast_for_suite(c, body);
if (!suite_seq)
return NULL;
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
return ExceptHandler(NULL, NULL, suite_seq, LINENO(exc),
- exc->n_col_offset, c->c_arena);
+ exc->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
else if (NCH(exc) == 2) {
expr_ty expression;
@@ -3759,9 +3877,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
suite_seq = ast_for_suite(c, body);
if (!suite_seq)
return NULL;
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
return ExceptHandler(expression, NULL, suite_seq, LINENO(exc),
- exc->n_col_offset, c->c_arena);
+ exc->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
else if (NCH(exc) == 4) {
asdl_seq *suite_seq;
@@ -3777,9 +3897,11 @@ ast_for_except_clause(struct compiling *c, const node *exc, node *body)
suite_seq = ast_for_suite(c, body);
if (!suite_seq)
return NULL;
+ get_last_end_pos(suite_seq, &end_lineno, &end_col_offset);
return ExceptHandler(expression, e, suite_seq, LINENO(exc),
- exc->n_col_offset, c->c_arena);
+ exc->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
PyErr_Format(PyExc_SystemError,
@@ -3792,8 +3914,9 @@ static stmt_ty
ast_for_try_stmt(struct compiling *c, const node *n)
{
const int nch = NCH(n);
- int n_except = (nch - 3)/3;
+ int end_lineno, end_col_offset, n_except = (nch - 3)/3;
asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL;
+ excepthandler_ty last_handler;
REQ(n, try_stmt);
@@ -3849,7 +3972,20 @@ ast_for_try_stmt(struct compiling *c, const node *n)
}
assert(finally != NULL || asdl_seq_LEN(handlers));
- return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c_arena);
+ if (finally != NULL) {
+ // finally is always last
+ get_last_end_pos(finally, &end_lineno, &end_col_offset);
+ } else if (orelse != NULL) {
+ // otherwise else is last
+ get_last_end_pos(orelse, &end_lineno, &end_col_offset);
+ } else {
+ // inline the get_last_end_pos logic due to layout mismatch
+ last_handler = (excepthandler_ty) asdl_seq_GET(handlers, n_except - 1);
+ end_lineno = last_handler->end_lineno;
+ end_col_offset = last_handler->end_col_offset;
+ }
+ return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
/* with_item: test ['as' expr] */
@@ -3881,7 +4017,7 @@ static stmt_ty
ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
{
const node * const n = is_async ? CHILD(n0, 1) : n0;
- int i, n_items;
+ int i, n_items, end_lineno, end_col_offset;
asdl_seq *items, *body;
REQ(n, with_stmt);
@@ -3900,11 +4036,14 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async)
body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
if (!body)
return NULL;
+ get_last_end_pos(body, &end_lineno, &end_col_offset);
if (is_async)
- return AsyncWith(items, body, LINENO(n0), n0->n_col_offset, c->c_arena);
+ return AsyncWith(items, body, LINENO(n0), n0->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
else
- return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
+ return With(items, body, LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
static stmt_ty
@@ -3914,6 +4053,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
PyObject *classname;
asdl_seq *s;
expr_ty call;
+ int end_lineno, end_col_offset;
REQ(n, classdef);
@@ -3921,26 +4061,32 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
s = ast_for_suite(c, CHILD(n, 3));
if (!s)
return NULL;
+ get_last_end_pos(s, &end_lineno, &end_col_offset);
+
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
if (forbidden_name(c, classname, CHILD(n, 3), 0))
return NULL;
return ClassDef(classname, NULL, NULL, s, decorator_seq,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
s = ast_for_suite(c, CHILD(n, 5));
if (!s)
return NULL;
+ get_last_end_pos(s, &end_lineno, &end_col_offset);
+
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
if (forbidden_name(c, classname, CHILD(n, 3), 0))
return NULL;
return ClassDef(classname, NULL, NULL, s, decorator_seq,
- LINENO(n), n->n_col_offset, c->c_arena);
+ LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
/* class NAME '(' arglist ')' ':' suite */
@@ -3951,14 +4097,18 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
dummy_name = NEW_IDENTIFIER(CHILD(n, 1));
if (!dummy_name)
return NULL;
- dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena);
- call = ast_for_call(c, CHILD(n, 3), dummy, NULL);
+ dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset,
+ CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset,
+ c->c_arena);
+ call = ast_for_call(c, CHILD(n, 3), dummy, NULL, CHILD(n, 4));
if (!call)
return NULL;
}
s = ast_for_suite(c, CHILD(n, 6));
if (!s)
return NULL;
+ get_last_end_pos(s, &end_lineno, &end_col_offset);
+
classname = NEW_IDENTIFIER(CHILD(n, 1));
if (!classname)
return NULL;
@@ -3966,7 +4116,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
return NULL;
return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s,
- decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
+ decorator_seq, LINENO(n), n->n_col_offset,
+ end_lineno, end_col_offset, c->c_arena);
}
static stmt_ty
@@ -3991,7 +4142,8 @@ ast_for_stmt(struct compiling *c, const node *n)
case del_stmt:
return ast_for_del_stmt(c, n);
case pass_stmt:
- return Pass(LINENO(n), n->n_col_offset, c->c_arena);
+ return Pass(LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case flow_stmt:
return ast_for_flow_stmt(c, n);
case import_stmt:
@@ -4248,6 +4400,7 @@ decode_bytes_with_escapes(struct compiling *c, const node *n, const char *s,
static void fstring_shift_node_locations(node *n, int lineno, int col_offset)
{
n->n_col_offset = n->n_col_offset + col_offset;
+ n->n_end_col_offset = n->n_end_col_offset + col_offset;
for (int i = 0; i < NCH(n); ++i) {
if (n->n_lineno && n->n_lineno < CHILD(n, i)->n_lineno) {
/* Shifting column offsets unnecessary if there's been newlines. */
@@ -4256,6 +4409,7 @@ static void fstring_shift_node_locations(node *n, int lineno, int col_offset)
fstring_shift_node_locations(CHILD(n, i), lineno, col_offset);
}
n->n_lineno = n->n_lineno + lineno;
+ n->n_end_lineno = n->n_end_lineno + lineno;
}
/* Fix locations for the given node and its children.
@@ -4672,6 +4826,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
entire expression with the conversion and format spec. */
*expression = FormattedValue(simple_expression, conversion,
format_spec, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
if (!*expression)
return -1;
@@ -4918,7 +5073,8 @@ make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
Py_DECREF(s);
return NULL;
}
- return Constant(s, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(s, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
/* Add a non-f-string (that is, a regular literal string). str is
@@ -5073,7 +5229,8 @@ FstringParser_Finish(FstringParser *state, struct compiling *c,
if (!seq)
goto error;
- return JoinedStr(seq, LINENO(n), n->n_col_offset, c->c_arena);
+ return JoinedStr(seq, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
error:
FstringParser_Dealloc(state);
@@ -5283,7 +5440,8 @@ parsestrplus(struct compiling *c, const node *n)
/* Just return the bytes object and we're done. */
if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0)
goto error;
- return Constant(bytes_str, LINENO(n), n->n_col_offset, c->c_arena);
+ return Constant(bytes_str, LINENO(n), n->n_col_offset,
+ n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
/* We're not a bytes string, bytes_str should never have been set. */
diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index 6f72a7f..96c766f 100644
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -439,7 +439,8 @@ astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
return 0;
}
asdl_seq_SET(values, 0, st->v.Expr.value);
- expr_ty expr = JoinedStr(values, st->lineno, st->col_offset, ctx_);
+ expr_ty expr = JoinedStr(values, st->lineno, st->col_offset,
+ st->end_lineno, st->end_col_offset, ctx_);
if (!expr) {
return 0;
}
diff --git a/Python/compile.c b/Python/compile.c
index 5aebda0..9713bfc 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -4757,7 +4757,8 @@ compiler_augassign(struct compiler *c, stmt_ty s)
switch (e->kind) {
case Attribute_kind:
auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
- AugLoad, e->lineno, e->col_offset, c->c_arena);
+ AugLoad, e->lineno, e->col_offset,
+ e->end_lineno, e->end_col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
@@ -4768,7 +4769,8 @@ compiler_augassign(struct compiler *c, stmt_ty s)
break;
case Subscript_kind:
auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
- AugLoad, e->lineno, e->col_offset, c->c_arena);
+ AugLoad, e->lineno, e->col_offset,
+ e->end_lineno, e->end_col_offset, c->c_arena);
if (auge == NULL)
return 0;
VISIT(c, expr, auge);
generated by cgit v0.12 at 2024-11-23 04:16:22 (GMT)