bpo-33416: Add end positions to Python AST (GH-11605)

The majority of this PR is tediously passing `end_lineno` and `end_col_offset` everywhere. Here are non-trivial points:
* It is not possible to reconstruct end positions in AST "on the fly", some information is lost after an AST node is constructed, so we need two more attributes for every AST node `end_lineno` and `end_col_offset`.
* I add end position information to both CST and AST.  Although it may be technically possible to avoid adding end positions to CST, the code becomes more cumbersome and less efficient.
* Since the end position is not known for non-leaf CST nodes while the next token is added, this requires a bit of extra care (see `_PyNode_FinalizeEndPos`). Unless I made some mistake, the algorithm should be linear.
* For statements, I "trim" the end position of suites to not include the terminal newlines and dedent (this seems to be what people would expect), for example in
  ```python
  class C:
      pass

  pass
  ```
  the end line and end column for the class definition is (2, 8).
* For `end_col_offset` I use the common Python convention for indexing, for example for `pass` the `end_col_offset` is 4 (not 3), so that `[0:4]` gives one the source code that corresponds to the node.
* I added a helper function `ast.get_source_segment()`, to get source text segment corresponding to a given AST node. It is also useful for testing.

An (inevitable) downside of this PR is that AST now takes almost 25% more memory. I think however it is probably justified by the benefits.

2 files changed, 165 insertions, 115 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 1a2b829..f8394e6 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -210,6 +210,8 @@ struct _stmt {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
@@ -353,6 +355,8 @@ struct _expr {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3};
@@ -396,6 +400,8 @@ struct _excepthandler {
     } v;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 struct _arguments {
@@ -412,6 +418,8 @@ struct _arg {
     expr_ty annotation;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 };
 
 struct _keyword {
@@ -430,6 +438,7 @@ struct _withitem {
 };
 
 
+// Note: these macros affect function definitions, not only call sites.
 #define Module(a0, a1) _Py_Module(a0, a1)
 mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
@@ -438,152 +447,188 @@ mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
                         asdl_seq * decorator_list, expr_ty returns, int lineno,
-                        int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                        int col_offset, int end_lineno, int end_col_offset,
+                        PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             int lineno, int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                             int lineno, int col_offset, int end_lineno, int
+                             end_col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
                      asdl_seq * body, asdl_seq * decorator_list, int lineno,
-                     int col_offset, PyArena *arena);
-#define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
-stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
-stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, PyArena
-                   *arena);
-#define Assign(a0, a1, a2, a3, a4) _Py_Assign(a0, a1, a2, a3, a4)
+                     int col_offset, int end_lineno, int end_col_offset,
+                     PyArena *arena);
+#define Return(a0, a1, a2, a3, a4, a5) _Py_Return(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno,
+                   int end_col_offset, PyArena *arena);
+#define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Delete(asdl_seq * targets, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Assign(a0, a1, a2, a3, a4, a5, a6) _Py_Assign(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_Assign(asdl_seq * targets, expr_ty value, int lineno, int
-                   col_offset, PyArena *arena);
-#define AugAssign(a0, a1, a2, a3, a4, a5) _Py_AugAssign(a0, a1, a2, a3, a4, a5)
+                   col_offset, int end_lineno, int end_col_offset, PyArena
+                   *arena);
+#define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int
-                      lineno, int col_offset, PyArena *arena);
-#define AnnAssign(a0, a1, a2, a3, a4, a5, a6) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int
-                      simple, int lineno, int col_offset, PyArena *arena);
-#define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6)
+                      simple, int lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
-                orelse, int lineno, int col_offset, PyArena *arena);
-#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6)
+                orelse, int lineno, int col_offset, int end_lineno, int
+                end_col_offset, PyArena *arena);
+#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq *
-                     orelse, int lineno, int col_offset, PyArena *arena);
-#define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5)
+                     orelse, int lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-                  int col_offset, PyArena *arena);
-#define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5)
+                  int col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define If(a0, a1, a2, a3, a4, a5, a6, a7) _Py_If(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
-               int col_offset, PyArena *arena);
-#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4)
+               int col_offset, int end_lineno, int end_col_offset, PyArena
+               *arena);
+#define With(a0, a1, a2, a3, a4, a5, a6) _Py_With(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
-                 PyArena *arena);
-#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4)
+                 int end_lineno, int end_col_offset, PyArena *arena);
+#define AsyncWith(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6)
 stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int
-                      col_offset, PyArena *arena);
-#define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4)
-stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset,
-                  PyArena *arena);
-#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6)
+                      col_offset, int end_lineno, int end_col_offset, PyArena
+                      *arena);
+#define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int
+                  end_lineno, int end_col_offset, PyArena *arena);
+#define Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_Try(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse,
-                asdl_seq * finalbody, int lineno, int col_offset, PyArena
-                *arena);
-#define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4)
-stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset,
-                   PyArena *arena);
-#define Import(a0, a1, a2, a3) _Py_Import(a0, a1, a2, a3)
-stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, PyArena
-                   *arena);
-#define ImportFrom(a0, a1, a2, a3, a4, a5) _Py_ImportFrom(a0, a1, a2, a3, a4, a5)
+                asdl_seq * finalbody, int lineno, int col_offset, int
+                end_lineno, int end_col_offset, PyArena *arena);
+#define Assert(a0, a1, a2, a3, a4, a5, a6) _Py_Assert(a0, a1, a2, a3, a4, a5, a6)
+stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Import(a0, a1, a2, a3, a4, a5) _Py_Import(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Import(asdl_seq * names, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
-                       lineno, int col_offset, PyArena *arena);
-#define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
-stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
-                   *arena);
-#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3)
-stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena
-                     *arena);
-#define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
-stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)
-stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
-#define Break(a0, a1, a2) _Py_Break(a0, a1, a2)
-stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
-#define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
-stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
-#define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
+                       lineno, int col_offset, int end_lineno, int
+                       end_col_offset, PyArena *arena);
+#define Global(a0, a1, a2, a3, a4, a5) _Py_Global(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, int
+                   end_lineno, int end_col_offset, PyArena *arena);
+#define Nonlocal(a0, a1, a2, a3, a4, a5) _Py_Nonlocal(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, int
+                     end_lineno, int end_col_offset, PyArena *arena);
+#define Expr(a0, a1, a2, a3, a4, a5) _Py_Expr(a0, a1, a2, a3, a4, a5)
+stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int
+                 end_col_offset, PyArena *arena);
+#define Pass(a0, a1, a2, a3, a4) _Py_Pass(a0, a1, a2, a3, a4)
+stmt_ty _Py_Pass(int lineno, int col_offset, int end_lineno, int
+                 end_col_offset, PyArena *arena);
+#define Break(a0, a1, a2, a3, a4) _Py_Break(a0, a1, a2, a3, a4)
+stmt_ty _Py_Break(int lineno, int col_offset, int end_lineno, int
+                  end_col_offset, PyArena *arena);
+#define Continue(a0, a1, a2, a3, a4) _Py_Continue(a0, a1, a2, a3, a4)
+stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
-                   PyArena *arena);
-#define BinOp(a0, a1, a2, a3, a4, a5) _Py_BinOp(a0, a1, a2, a3, a4, a5)
+                   int end_lineno, int end_col_offset, PyArena *arena);
+#define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int
-                  col_offset, PyArena *arena);
-#define UnaryOp(a0, a1, a2, a3, a4) _Py_UnaryOp(a0, a1, a2, a3, a4)
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define UnaryOp(a0, a1, a2, a3, a4, a5, a6) _Py_UnaryOp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset,
-                    PyArena *arena);
-#define Lambda(a0, a1, a2, a3, a4) _Py_Lambda(a0, a1, a2, a3, a4)
+                    int end_lineno, int end_col_offset, PyArena *arena);
+#define Lambda(a0, a1, a2, a3, a4, a5, a6) _Py_Lambda(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset,
-                   PyArena *arena);
-#define IfExp(a0, a1, a2, a3, a4, a5) _Py_IfExp(a0, a1, a2, a3, a4, a5)
+                   int end_lineno, int end_col_offset, PyArena *arena);
+#define IfExp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_IfExp(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int
-                  col_offset, PyArena *arena);
-#define Dict(a0, a1, a2, a3, a4) _Py_Dict(a0, a1, a2, a3, a4)
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
+#define Dict(a0, a1, a2, a3, a4, a5, a6) _Py_Dict(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Dict(asdl_seq * keys, asdl_seq * values, int lineno, int
-                 col_offset, PyArena *arena);
-#define Set(a0, a1, a2, a3) _Py_Set(a0, a1, a2, a3)
-expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena);
-#define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define Set(a0, a1, a2, a3, a4, a5) _Py_Set(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno,
+                int end_col_offset, PyArena *arena);
+#define ListComp(a0, a1, a2, a3, a4, a5, a6) _Py_ListComp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int
-                     col_offset, PyArena *arena);
-#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4)
+                     col_offset, int end_lineno, int end_col_offset, PyArena
+                     *arena);
+#define SetComp(a0, a1, a2, a3, a4, a5, a6) _Py_SetComp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int
-                    col_offset, PyArena *arena);
-#define DictComp(a0, a1, a2, a3, a4, a5) _Py_DictComp(a0, a1, a2, a3, a4, a5)
+                    col_offset, int end_lineno, int end_col_offset, PyArena
+                    *arena);
+#define DictComp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_DictComp(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
-                     lineno, int col_offset, PyArena *arena);
-#define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
+                     lineno, int col_offset, int end_lineno, int
+                     end_col_offset, PyArena *arena);
+#define GeneratorExp(a0, a1, a2, a3, a4, a5, a6) _Py_GeneratorExp(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
-                         col_offset, PyArena *arena);
-#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
-expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
-expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
-#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)
-expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
-                      *arena);
-#define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
+                         col_offset, int end_lineno, int end_col_offset,
+                         PyArena *arena);
+#define Await(a0, a1, a2, a3, a4, a5) _Py_Await(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, int end_lineno,
+                  int end_col_offset, PyArena *arena);
+#define Yield(a0, a1, a2, a3, a4, a5) _Py_Yield(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, int end_lineno,
+                  int end_col_offset, PyArena *arena);
+#define YieldFrom(a0, a1, a2, a3, a4, a5) _Py_YieldFrom(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, int
+                      end_lineno, int end_col_offset, PyArena *arena);
+#define Compare(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Compare(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
-                    int lineno, int col_offset, PyArena *arena);
-#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5)
+                    int lineno, int col_offset, int end_lineno, int
+                    end_col_offset, PyArena *arena);
+#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
-                 lineno, int col_offset, PyArena *arena);
-#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5)
+                 lineno, int col_offset, int end_lineno, int end_col_offset,
+                 PyArena *arena);
+#define FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
-                           int lineno, int col_offset, PyArena *arena);
-#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3)
-expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena
-                      *arena);
-#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3)
-expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena
-                     *arena);
-#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
+                           int lineno, int col_offset, int end_lineno, int
+                           end_col_offset, PyArena *arena);
+#define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, int
+                      end_lineno, int end_col_offset, PyArena *arena);
+#define Constant(a0, a1, a2, a3, a4, a5) _Py_Constant(a0, a1, a2, a3, a4, a5)
+expr_ty _Py_Constant(constant value, int lineno, int col_offset, int
+                     end_lineno, int end_col_offset, PyArena *arena);
+#define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
-                      lineno, int col_offset, PyArena *arena);
-#define Subscript(a0, a1, a2, a3, a4, a5) _Py_Subscript(a0, a1, a2, a3, a4, a5)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7)
 expr_ty _Py_Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
-                      lineno, int col_offset, PyArena *arena);
-#define Starred(a0, a1, a2, a3, a4) _Py_Starred(a0, a1, a2, a3, a4)
+                      lineno, int col_offset, int end_lineno, int
+                      end_col_offset, PyArena *arena);
+#define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Starred(expr_ty value, expr_context_ty ctx, int lineno, int
-                    col_offset, PyArena *arena);
-#define Name(a0, a1, a2, a3, a4) _Py_Name(a0, a1, a2, a3, a4)
+                    col_offset, int end_lineno, int end_col_offset, PyArena
+                    *arena);
+#define Name(a0, a1, a2, a3, a4, a5, a6) _Py_Name(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int
-                 col_offset, PyArena *arena);
-#define List(a0, a1, a2, a3, a4) _Py_List(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define List(a0, a1, a2, a3, a4, a5, a6) _Py_List(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_List(asdl_seq * elts, expr_context_ty ctx, int lineno, int
-                 col_offset, PyArena *arena);
-#define Tuple(a0, a1, a2, a3, a4) _Py_Tuple(a0, a1, a2, a3, a4)
+                 col_offset, int end_lineno, int end_col_offset, PyArena
+                 *arena);
+#define Tuple(a0, a1, a2, a3, a4, a5, a6) _Py_Tuple(a0, a1, a2, a3, a4, a5, a6)
 expr_ty _Py_Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int
-                  col_offset, PyArena *arena);
+                  col_offset, int end_lineno, int end_col_offset, PyArena
+                  *arena);
 #define Slice(a0, a1, a2, a3) _Py_Slice(a0, a1, a2, a3)
 slice_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
 #define ExtSlice(a0, a1) _Py_ExtSlice(a0, a1)
@@ -593,17 +638,18 @@ slice_ty _Py_Index(expr_ty value, PyArena *arena);
 #define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4)
 comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
                                    ifs, int is_async, PyArena *arena);
-#define ExceptHandler(a0, a1, a2, a3, a4, a5) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5)
+#define ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7)
 excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
-                                   body, int lineno, int col_offset, PyArena
+                                   body, int lineno, int col_offset, int
+                                   end_lineno, int end_col_offset, PyArena
                                    *arena);
 #define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
 arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
                            kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
                            asdl_seq * defaults, PyArena *arena);
-#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4)
+#define arg(a0, a1, a2, a3, a4, a5, a6) _Py_arg(a0, a1, a2, a3, a4, a5, a6)
 arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset,
-               PyArena *arena);
+               int end_lineno, int end_col_offset, PyArena *arena);
 #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
 keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
diff --git a/Include/node.h b/Include/node.h
index 40596df..2b39074 100644
--- a/Include/node.h
+++ b/Include/node.h
@@ -14,11 +14,14 @@ typedef struct _node {
     int                 n_col_offset;
     int                 n_nchildren;
     struct _node        *n_child;
+    int                 n_end_lineno;
+    int                 n_end_col_offset;
 } node;
 
 PyAPI_FUNC(node *) PyNode_New(int type);
 PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
-                                      char *str, int lineno, int col_offset);
+                                char *str, int lineno, int col_offset,
+                                int end_lineno, int end_col_offset);
 PyAPI_FUNC(void) PyNode_Free(node *n);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
@@ -37,6 +40,7 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n);
 #define REQ(n, type) assert(TYPE(n) == (type))
 
 PyAPI_FUNC(void) PyNode_ListTree(node *);
+void _PyNode_FinalizeEndPos(node *n);  // helper also used in parsetok.c
 
 #ifdef __cplusplus
 }