bpo-33416: Add end positions to Python AST (GH-11605)

The majority of this PR is tediously passing `end_lineno` and `end_col_offset` everywhere. Here are non-trivial points:
* It is not possible to reconstruct end positions in AST "on the fly", some information is lost after an AST node is constructed, so we need two more attributes for every AST node `end_lineno` and `end_col_offset`.
* I add end position information to both CST and AST.  Although it may be technically possible to avoid adding end positions to CST, the code becomes more cumbersome and less efficient.
* Since the end position is not known for non-leaf CST nodes while the next token is added, this requires a bit of extra care (see `_PyNode_FinalizeEndPos`). Unless I made some mistake, the algorithm should be linear.
* For statements, I "trim" the end position of suites to not include the terminal newlines and dedent (this seems to be what people would expect), for example in
  ```python
  class C:
      pass

  pass
  ```
  the end line and end column for the class definition is (2, 8).
* For `end_col_offset` I use the common Python convention for indexing, for example for `pass` the `end_col_offset` is 4 (not 3), so that `[0:4]` gives one the source code that corresponds to the node.
* I added a helper function `ast.get_source_segment()`, to get source text segment corresponding to a given AST node. It is also useful for testing.

An (inevitable) downside of this PR is that AST now takes almost 25% more memory. I think however it is probably justified by the benefits.

1 files changed, 454 insertions, 124 deletions

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index bbe8e69fdd..e6c5bfe9b2 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -28,9 +28,13 @@ static char *Suite_fields[]={
 static PyTypeObject *stmt_type;
 _Py_IDENTIFIER(lineno);
 _Py_IDENTIFIER(col_offset);
+_Py_IDENTIFIER(end_lineno);
+_Py_IDENTIFIER(end_col_offset);
 static char *stmt_attributes[] = {
     "lineno",
     "col_offset",
+    "end_lineno",
+    "end_col_offset",
 };
 static PyObject* ast2obj_stmt(void*);
 static PyTypeObject *FunctionDef_type;
@@ -189,6 +193,8 @@ static PyTypeObject *expr_type;
 static char *expr_attributes[] = {
     "lineno",
     "col_offset",
+    "end_lineno",
+    "end_col_offset",
 };
 static PyObject* ast2obj_expr(void*);
 static PyTypeObject *BoolOp_type;
@@ -427,6 +433,8 @@ static PyTypeObject *excepthandler_type;
 static char *excepthandler_attributes[] = {
     "lineno",
     "col_offset",
+    "end_lineno",
+    "end_col_offset",
 };
 static PyObject* ast2obj_excepthandler(void*);
 static PyTypeObject *ExceptHandler_type;
@@ -456,6 +464,8 @@ static PyObject* ast2obj_arg(void*);
 static char *arg_attributes[] = {
     "lineno",
     "col_offset",
+    "end_lineno",
+    "end_col_offset",
 };
 _Py_IDENTIFIER(arg);
 static char *arg_fields[]={
@@ -804,7 +814,7 @@ static int init_types(void)
     if (!Suite_type) return 0;
     stmt_type = make_type("stmt", &AST_type, NULL, 0);
     if (!stmt_type) return 0;
-    if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0;
+    if (!add_attributes(stmt_type, stmt_attributes, 4)) return 0;
     FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
                                  5);
     if (!FunctionDef_type) return 0;
@@ -859,7 +869,7 @@ static int init_types(void)
     if (!Continue_type) return 0;
     expr_type = make_type("expr", &AST_type, NULL, 0);
     if (!expr_type) return 0;
-    if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
+    if (!add_attributes(expr_type, expr_attributes, 4)) return 0;
     BoolOp_type = make_type("BoolOp", expr_type, BoolOp_fields, 2);
     if (!BoolOp_type) return 0;
     BinOp_type = make_type("BinOp", expr_type, BinOp_fields, 3);
@@ -1082,7 +1092,7 @@ static int init_types(void)
     if (!add_attributes(comprehension_type, NULL, 0)) return 0;
     excepthandler_type = make_type("excepthandler", &AST_type, NULL, 0);
     if (!excepthandler_type) return 0;
-    if (!add_attributes(excepthandler_type, excepthandler_attributes, 2))
+    if (!add_attributes(excepthandler_type, excepthandler_attributes, 4))
         return 0;
     ExceptHandler_type = make_type("ExceptHandler", excepthandler_type,
                                    ExceptHandler_fields, 3);
@@ -1092,7 +1102,7 @@ static int init_types(void)
     if (!add_attributes(arguments_type, NULL, 0)) return 0;
     arg_type = make_type("arg", &AST_type, arg_fields, 2);
     if (!arg_type) return 0;
-    if (!add_attributes(arg_type, arg_attributes, 2)) return 0;
+    if (!add_attributes(arg_type, arg_attributes, 4)) return 0;
     keyword_type = make_type("keyword", &AST_type, keyword_fields, 2);
     if (!keyword_type) return 0;
     if (!add_attributes(keyword_type, NULL, 0)) return 0;
@@ -1181,8 +1191,8 @@ Suite(asdl_seq * body, PyArena *arena)
 
 stmt_ty
 FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
-            decorator_list, expr_ty returns, int lineno, int col_offset,
-            PyArena *arena)
+            decorator_list, expr_ty returns, int lineno, int col_offset, int
+            end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1206,13 +1216,15 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
     p->v.FunctionDef.returns = returns;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
                  * decorator_list, expr_ty returns, int lineno, int col_offset,
-                 PyArena *arena)
+                 int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1236,13 +1248,15 @@ AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
     p->v.AsyncFunctionDef.returns = returns;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
-         body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
-         *arena)
+         body, asdl_seq * decorator_list, int lineno, int col_offset, int
+         end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1261,11 +1275,14 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
     p->v.ClassDef.decorator_list = decorator_list;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Return(expr_ty value, int lineno, int col_offset, int end_lineno, int
+       end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1275,11 +1292,14 @@ Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
     p->v.Return.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
+Delete(asdl_seq * targets, int lineno, int col_offset, int end_lineno, int
+       end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1289,12 +1309,14 @@ Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
     p->v.Delete.targets = targets;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
-       *arena)
+Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, int
+       end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!value) {
@@ -1310,12 +1332,14 @@ Assign(asdl_seq * targets, expr_ty value, int lineno, int col_offset, PyArena
     p->v.Assign.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
-          col_offset, PyArena *arena)
+          col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!target) {
@@ -1342,12 +1366,15 @@ AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
     p->v.AugAssign.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int
-          lineno, int col_offset, PyArena *arena)
+          lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+          *arena)
 {
     stmt_ty p;
     if (!target) {
@@ -1370,12 +1397,14 @@ AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int simple, int
     p->v.AnnAssign.simple = simple;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
-    lineno, int col_offset, PyArena *arena)
+    lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!target) {
@@ -1398,12 +1427,15 @@ For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
     p->v.For.orelse = orelse;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
-         lineno, int col_offset, PyArena *arena)
+         lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+         *arena)
 {
     stmt_ty p;
     if (!target) {
@@ -1426,12 +1458,14 @@ AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int
     p->v.AsyncFor.orelse = orelse;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
-      col_offset, PyArena *arena)
+      col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!test) {
@@ -1448,12 +1482,14 @@ While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
     p->v.While.orelse = orelse;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
-   col_offset, PyArena *arena)
+   col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!test) {
@@ -1470,12 +1506,14 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
     p->v.If.orelse = orelse;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
-     *arena)
+With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int
+     end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1486,12 +1524,14 @@ With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena
     p->v.With.body = body;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
-          PyArena *arena)
+AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, int
+          end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1502,11 +1542,14 @@ AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int col_offset,
     p->v.AsyncWith.body = body;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
+Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int end_lineno,
+      int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1517,12 +1560,15 @@ Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
     p->v.Raise.cause = cause;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq *
-    finalbody, int lineno, int col_offset, PyArena *arena)
+    finalbody, int lineno, int col_offset, int end_lineno, int end_col_offset,
+    PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1535,11 +1581,14 @@ Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq *
     p->v.Try.finalbody = finalbody;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
+Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno,
+       int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!test) {
@@ -1555,11 +1604,14 @@ Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena)
     p->v.Assert.msg = msg;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Import(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+       end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1569,12 +1621,14 @@ Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
     p->v.Import.names = names;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
 ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
-           col_offset, PyArena *arena)
+           col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1586,11 +1640,14 @@ ImportFrom(identifier module, asdl_seq * names, int level, int lineno, int
     p->v.ImportFrom.level = level;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Global(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+       end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1600,11 +1657,14 @@ Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
     p->v.Global.names = names;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+Nonlocal(asdl_seq * names, int lineno, int col_offset, int end_lineno, int
+         end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1614,11 +1674,14 @@ Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
     p->v.Nonlocal.names = names;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int
+     end_col_offset, PyArena *arena)
 {
     stmt_ty p;
     if (!value) {
@@ -1633,11 +1696,14 @@ Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
     p->v.Expr.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Pass(int lineno, int col_offset, PyArena *arena)
+Pass(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+     *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1646,11 +1712,14 @@ Pass(int lineno, int col_offset, PyArena *arena)
     p->kind = Pass_kind;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Break(int lineno, int col_offset, PyArena *arena)
+Break(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena
+      *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1659,11 +1728,14 @@ Break(int lineno, int col_offset, PyArena *arena)
     p->kind = Break_kind;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 stmt_ty
-Continue(int lineno, int col_offset, PyArena *arena)
+Continue(int lineno, int col_offset, int end_lineno, int end_col_offset,
+         PyArena *arena)
 {
     stmt_ty p;
     p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1672,12 +1744,14 @@ Continue(int lineno, int col_offset, PyArena *arena)
     p->kind = Continue_kind;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
-       *arena)
+BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, int
+       end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!op) {
@@ -1693,12 +1767,14 @@ BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
     p->v.BoolOp.values = values;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
-      PyArena *arena)
+      int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!left) {
@@ -1725,12 +1801,14 @@ BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
     p->v.BinOp.right = right;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
-        *arena)
+UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, int
+        end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!op) {
@@ -1751,12 +1829,14 @@ UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
     p->v.UnaryOp.operand = operand;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
-       *arena)
+Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, int
+       end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!args) {
@@ -1777,12 +1857,14 @@ Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
     p->v.Lambda.body = body;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
-      PyArena *arena)
+      int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!test) {
@@ -1809,12 +1891,14 @@ IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
     p->v.IfExp.orelse = orelse;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
-     *arena)
+Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, int
+     end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1825,11 +1909,14 @@ Dict(asdl_seq * keys, asdl_seq * values, int lineno, int col_offset, PyArena
     p->v.Dict.values = values;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena)
+Set(asdl_seq * elts, int lineno, int col_offset, int end_lineno, int
+    end_col_offset, PyArena *arena)
 {
     expr_ty p;
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1839,12 +1926,14 @@ Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena)
     p->v.Set.elts = elts;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
-         PyArena *arena)
+ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int
+         end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!elt) {
@@ -1860,12 +1949,14 @@ ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
     p->v.ListComp.generators = generators;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
-        *arena)
+SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, int
+        end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!elt) {
@@ -1881,12 +1972,14 @@ SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena
     p->v.SetComp.generators = generators;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int
-         col_offset, PyArena *arena)
+         col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!key) {
@@ -1908,12 +2001,14 @@ DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int lineno, int
     p->v.DictComp.generators = generators;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
-             PyArena *arena)
+             int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!elt) {
@@ -1929,11 +2024,14 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
     p->v.GeneratorExp.generators = generators;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Await(expr_ty value, int lineno, int col_offset, int end_lineno, int
+      end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -1948,11 +2046,14 @@ Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
     p->v.Await.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
+Yield(expr_ty value, int lineno, int col_offset, int end_lineno, int
+      end_col_offset, PyArena *arena)
 {
     expr_ty p;
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -1962,11 +2063,14 @@ Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
     p->v.Yield.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
+YieldFrom(expr_ty value, int lineno, int col_offset, int end_lineno, int
+          end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -1981,12 +2085,14 @@ YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
     p->v.YieldFrom.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
-        int col_offset, PyArena *arena)
+        int col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!left) {
@@ -2003,12 +2109,14 @@ Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
     p->v.Compare.comparators = comparators;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
-     col_offset, PyArena *arena)
+     col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!func) {
@@ -2025,12 +2133,15 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
     p->v.Call.keywords = keywords;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
-               int col_offset, PyArena *arena)
+               int col_offset, int end_lineno, int end_col_offset, PyArena
+               *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2047,11 +2158,14 @@ FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
     p->v.FormattedValue.format_spec = format_spec;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
+JoinedStr(asdl_seq * values, int lineno, int col_offset, int end_lineno, int
+          end_col_offset, PyArena *arena)
 {
     expr_ty p;
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2061,11 +2175,14 @@ JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
     p->v.JoinedStr.values = values;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Constant(constant value, int lineno, int col_offset, PyArena *arena)
+Constant(constant value, int lineno, int col_offset, int end_lineno, int
+         end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2080,12 +2197,14 @@ Constant(constant value, int lineno, int col_offset, PyArena *arena)
     p->v.Constant.value = value;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
-          col_offset, PyArena *arena)
+          col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2112,12 +2231,14 @@ Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
     p->v.Attribute.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
 Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
-          col_offset, PyArena *arena)
+          col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2144,12 +2265,14 @@ Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int lineno, int
     p->v.Subscript.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena
-        *arena)
+Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, int
+        end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2170,12 +2293,14 @@ Starred(expr_ty value, expr_context_ty ctx, int lineno, int col_offset, PyArena
     p->v.Starred.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
-     *arena)
+Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, int
+     end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!id) {
@@ -2196,12 +2321,14 @@ Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
     p->v.Name.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
-     *arena)
+List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int
+     end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!ctx) {
@@ -2217,12 +2344,14 @@ List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
     p->v.List.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
 expr_ty
-Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
-      *arena)
+Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, int
+      end_lineno, int end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!ctx) {
@@ -2238,6 +2367,8 @@ Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
     p->v.Tuple.ctx = ctx;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
@@ -2311,7 +2442,7 @@ comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async,
 
 excepthandler_ty
 ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int
-              col_offset, PyArena *arena)
+              col_offset, int end_lineno, int end_col_offset, PyArena *arena)
 {
     excepthandler_ty p;
     p = (excepthandler_ty)PyArena_Malloc(arena, sizeof(*p));
@@ -2323,6 +2454,8 @@ ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int
     p->v.ExceptHandler.body = body;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
@@ -2344,8 +2477,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq *
 }
 
 arg_ty
-arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
-    *arena)
+arg(identifier arg, expr_ty annotation, int lineno, int col_offset, int
+    end_lineno, int end_col_offset, PyArena *arena)
 {
     arg_ty p;
     if (!arg) {
@@ -2360,6 +2493,8 @@ arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
     p->annotation = annotation;
     p->lineno = lineno;
     p->col_offset = col_offset;
+    p->end_lineno = end_lineno;
+    p->end_col_offset = end_col_offset;
     return p;
 }
 
@@ -2886,6 +3021,16 @@ ast2obj_stmt(void* _o)
     if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
+    value = ast2obj_int(o->end_lineno);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+        goto failed;
+    Py_DECREF(value);
+    value = ast2obj_int(o->end_col_offset);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+        goto failed;
+    Py_DECREF(value);
     return result;
 failed:
     Py_XDECREF(value);
@@ -3281,6 +3426,16 @@ ast2obj_expr(void* _o)
     if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
+    value = ast2obj_int(o->end_lineno);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+        goto failed;
+    Py_DECREF(value);
+    value = ast2obj_int(o->end_col_offset);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+        goto failed;
+    Py_DECREF(value);
     return result;
 failed:
     Py_XDECREF(value);
@@ -3571,6 +3726,16 @@ ast2obj_excepthandler(void* _o)
     if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
+    value = ast2obj_int(o->end_lineno);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+        goto failed;
+    Py_DECREF(value);
+    value = ast2obj_int(o->end_col_offset);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+        goto failed;
+    Py_DECREF(value);
     return result;
 failed:
     Py_XDECREF(value);
@@ -3657,6 +3822,16 @@ ast2obj_arg(void* _o)
     if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
         goto failed;
     Py_DECREF(value);
+    value = ast2obj_int(o->end_lineno);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_lineno, value) < 0)
+        goto failed;
+    Py_DECREF(value);
+    value = ast2obj_int(o->end_col_offset);
+    if (!value) goto failed;
+    if (_PyObject_SetAttrId(result, &PyId_end_col_offset, value) < 0)
+        goto failed;
+    Py_DECREF(value);
     return result;
 failed:
     Py_XDECREF(value);
@@ -3922,6 +4097,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
     PyObject *tmp = NULL;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 
     if (obj == Py_None) {
         *out = NULL;
@@ -3953,6 +4130,32 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_lineno = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_col_offset = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionDef_type);
     if (isinstance == -1) {
         return 1;
@@ -4064,7 +4267,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = FunctionDef(name, args, body, decorator_list, returns, lineno,
-                           col_offset, arena);
+                           col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4179,7 +4382,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = AsyncFunctionDef(name, args, body, decorator_list, returns,
-                                lineno, col_offset, arena);
+                                lineno, col_offset, end_lineno, end_col_offset,
+                                arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4328,7 +4532,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = ClassDef(name, bases, keywords, body, decorator_list, lineno,
-                        col_offset, arena);
+                        col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4352,7 +4556,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Return(value, lineno, col_offset, arena);
+        *out = Return(value, lineno, col_offset, end_lineno, end_col_offset,
+                      arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4393,7 +4598,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Delete(targets, lineno, col_offset, arena);
+        *out = Delete(targets, lineno, col_offset, end_lineno, end_col_offset,
+                      arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4448,7 +4654,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Assign(targets, value, lineno, col_offset, arena);
+        *out = Assign(targets, value, lineno, col_offset, end_lineno,
+                      end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4500,7 +4707,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = AugAssign(target, op, value, lineno, col_offset, arena);
+        *out = AugAssign(target, op, value, lineno, col_offset, end_lineno,
+                         end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4567,7 +4775,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = AnnAssign(target, annotation, value, simple, lineno, col_offset,
-                         arena);
+                         end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4667,7 +4875,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = For(target, iter, body, orelse, lineno, col_offset, arena);
+        *out = For(target, iter, body, orelse, lineno, col_offset, end_lineno,
+                   end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4767,7 +4976,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena);
+        *out = AsyncFor(target, iter, body, orelse, lineno, col_offset,
+                        end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4853,7 +5063,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = While(test, body, orelse, lineno, col_offset, arena);
+        *out = While(test, body, orelse, lineno, col_offset, end_lineno,
+                     end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -4939,7 +5150,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = If(test, body, orelse, lineno, col_offset, arena);
+        *out = If(test, body, orelse, lineno, col_offset, end_lineno,
+                  end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5011,7 +5223,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = With(items, body, lineno, col_offset, arena);
+        *out = With(items, body, lineno, col_offset, end_lineno,
+                    end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5083,7 +5296,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = AsyncWith(items, body, lineno, col_offset, arena);
+        *out = AsyncWith(items, body, lineno, col_offset, end_lineno,
+                         end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5121,7 +5335,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Raise(exc, cause, lineno, col_offset, arena);
+        *out = Raise(exc, cause, lineno, col_offset, end_lineno,
+                     end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5256,7 +5471,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = Try(body, handlers, orelse, finalbody, lineno, col_offset,
-                   arena);
+                   end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5294,7 +5509,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Assert(test, msg, lineno, col_offset, arena);
+        *out = Assert(test, msg, lineno, col_offset, end_lineno,
+                      end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5335,7 +5551,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Import(names, lineno, col_offset, arena);
+        *out = Import(names, lineno, col_offset, end_lineno, end_col_offset,
+                      arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5404,7 +5621,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = ImportFrom(module, names, level, lineno, col_offset, arena);
+        *out = ImportFrom(module, names, level, lineno, col_offset, end_lineno,
+                          end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5445,7 +5663,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Global(names, lineno, col_offset, arena);
+        *out = Global(names, lineno, col_offset, end_lineno, end_col_offset,
+                      arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5486,7 +5705,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Nonlocal(names, lineno, col_offset, arena);
+        *out = Nonlocal(names, lineno, col_offset, end_lineno, end_col_offset,
+                        arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5510,7 +5730,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Expr(value, lineno, col_offset, arena);
+        *out = Expr(value, lineno, col_offset, end_lineno, end_col_offset,
+                    arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5520,7 +5741,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
     }
     if (isinstance) {
 
-        *out = Pass(lineno, col_offset, arena);
+        *out = Pass(lineno, col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5530,7 +5751,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
     }
     if (isinstance) {
 
-        *out = Break(lineno, col_offset, arena);
+        *out = Break(lineno, col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5540,7 +5761,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
     }
     if (isinstance) {
 
-        *out = Continue(lineno, col_offset, arena);
+        *out = Continue(lineno, col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5559,6 +5780,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
     PyObject *tmp = NULL;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 
     if (obj == Py_None) {
         *out = NULL;
@@ -5590,6 +5813,32 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_lineno = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_col_offset = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)BoolOp_type);
     if (isinstance == -1) {
         return 1;
@@ -5641,7 +5890,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = BoolOp(op, values, lineno, col_offset, arena);
+        *out = BoolOp(op, values, lineno, col_offset, end_lineno,
+                      end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5693,7 +5943,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = BinOp(left, op, right, lineno, col_offset, arena);
+        *out = BinOp(left, op, right, lineno, col_offset, end_lineno,
+                     end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5731,7 +5982,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = UnaryOp(op, operand, lineno, col_offset, arena);
+        *out = UnaryOp(op, operand, lineno, col_offset, end_lineno,
+                       end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5769,7 +6021,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Lambda(args, body, lineno, col_offset, arena);
+        *out = Lambda(args, body, lineno, col_offset, end_lineno,
+                      end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5821,7 +6074,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = IfExp(test, body, orelse, lineno, col_offset, arena);
+        *out = IfExp(test, body, orelse, lineno, col_offset, end_lineno,
+                     end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5893,7 +6147,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Dict(keys, values, lineno, col_offset, arena);
+        *out = Dict(keys, values, lineno, col_offset, end_lineno,
+                    end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5934,7 +6189,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Set(elts, lineno, col_offset, arena);
+        *out = Set(elts, lineno, col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5989,7 +6244,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = ListComp(elt, generators, lineno, col_offset, arena);
+        *out = ListComp(elt, generators, lineno, col_offset, end_lineno,
+                        end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6044,7 +6300,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = SetComp(elt, generators, lineno, col_offset, arena);
+        *out = SetComp(elt, generators, lineno, col_offset, end_lineno,
+                       end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6113,7 +6370,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = DictComp(key, value, generators, lineno, col_offset, arena);
+        *out = DictComp(key, value, generators, lineno, col_offset, end_lineno,
+                        end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6168,7 +6426,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = GeneratorExp(elt, generators, lineno, col_offset, arena);
+        *out = GeneratorExp(elt, generators, lineno, col_offset, end_lineno,
+                            end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6192,7 +6451,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Await(value, lineno, col_offset, arena);
+        *out = Await(value, lineno, col_offset, end_lineno, end_col_offset,
+                     arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6216,7 +6476,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Yield(value, lineno, col_offset, arena);
+        *out = Yield(value, lineno, col_offset, end_lineno, end_col_offset,
+                     arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6240,7 +6501,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = YieldFrom(value, lineno, col_offset, arena);
+        *out = YieldFrom(value, lineno, col_offset, end_lineno, end_col_offset,
+                         arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6326,7 +6588,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Compare(left, ops, comparators, lineno, col_offset, arena);
+        *out = Compare(left, ops, comparators, lineno, col_offset, end_lineno,
+                       end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6412,7 +6675,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = Call(func, args, keywords, lineno, col_offset, arena);
+        *out = Call(func, args, keywords, lineno, col_offset, end_lineno,
+                    end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6465,7 +6729,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             Py_CLEAR(tmp);
         }
         *out = FormattedValue(value, conversion, format_spec, lineno,
-                              col_offset, arena);
+                              col_offset, end_lineno, end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6506,7 +6770,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = JoinedStr(values, lineno, col_offset, arena);
+        *out = JoinedStr(values, lineno, col_offset, end_lineno,
+                         end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6530,7 +6795,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Constant(value, lineno, col_offset, arena);
+        *out = Constant(value, lineno, col_offset, end_lineno, end_col_offset,
+                        arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6582,7 +6848,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Attribute(value, attr, ctx, lineno, col_offset, arena);
+        *out = Attribute(value, attr, ctx, lineno, col_offset, end_lineno,
+                         end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6634,7 +6901,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Subscript(value, slice, ctx, lineno, col_offset, arena);
+        *out = Subscript(value, slice, ctx, lineno, col_offset, end_lineno,
+                         end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6672,7 +6940,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Starred(value, ctx, lineno, col_offset, arena);
+        *out = Starred(value, ctx, lineno, col_offset, end_lineno,
+                       end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6710,7 +6979,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Name(id, ctx, lineno, col_offset, arena);
+        *out = Name(id, ctx, lineno, col_offset, end_lineno, end_col_offset,
+                    arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6765,7 +7035,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = List(elts, ctx, lineno, col_offset, arena);
+        *out = List(elts, ctx, lineno, col_offset, end_lineno, end_col_offset,
+                    arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6820,7 +7091,8 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Tuple(elts, ctx, lineno, col_offset, arena);
+        *out = Tuple(elts, ctx, lineno, col_offset, end_lineno, end_col_offset,
+                     arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -7389,6 +7661,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
     PyObject *tmp = NULL;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 
     if (obj == Py_None) {
         *out = NULL;
@@ -7420,6 +7694,32 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_lineno = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_col_offset = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type);
     if (isinstance == -1) {
         return 1;
@@ -7485,7 +7785,8 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena)
             }
             Py_CLEAR(tmp);
         }
-        *out = ExceptHandler(type, name, body, lineno, col_offset, arena);
+        *out = ExceptHandler(type, name, body, lineno, col_offset, end_lineno,
+                             end_col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -7669,6 +7970,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     expr_ty annotation;
     int lineno;
     int col_offset;
+    int end_lineno;
+    int end_col_offset;
 
     if (_PyObject_LookupAttrId(obj, &PyId_arg, &tmp) < 0) {
         return 1;
@@ -7722,7 +8025,34 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     }
-    *out = arg(arg, annotation, lineno, col_offset, arena);
+    if (_PyObject_LookupAttrId(obj, &PyId_end_lineno, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_lineno = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
+    if (_PyObject_LookupAttrId(obj, &PyId_end_col_offset, &tmp) < 0) {
+        return 1;
+    }
+    if (tmp == NULL || tmp == Py_None) {
+        Py_CLEAR(tmp);
+        end_col_offset = 0;
+    }
+    else {
+        int res;
+        res = obj2ast_int(tmp, &end_col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    }
+    *out = arg(arg, annotation, lineno, col_offset, end_lineno, end_col_offset,
+               arena);
     return 0;
 failed:
     Py_XDECREF(tmp);