Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node when

compiling AST from Python objects.

5 files changed, 48 insertions, 12 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index ea6679c..175e380 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -602,8 +602,9 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
 arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
                            kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
                            asdl_seq * defaults, PyArena *arena);
-#define arg(a0, a1, a2) _Py_arg(a0, a1, a2)
-arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
+#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4)
+arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset,
+               PyArena *arena);
 #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
 keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
diff --git a/Misc/NEWS b/Misc/NEWS
index e42df93..68de958 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@ Release date: XXXX-XX-XX
 Core and Builtins
 -----------------
 
+- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node
+  when compiling AST from Python objects.
+
 - Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that
   was mutated by direct calls of dict methods.
 
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 3128078..eedd89b 100644
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -275,7 +275,9 @@ class PrototypeVisitor(EmitVisitor):
 
     def visitProduct(self, prod, name):
         self.emit_function(name, get_c_type(name),
-                           self.get_args(prod.fields), [], union=False)
+                           self.get_args(prod.fields),
+                           self.get_args(prod.attributes),
+                           union=False)
 
 
 class FunctionVisitor(PrototypeVisitor):
@@ -329,7 +331,8 @@ class FunctionVisitor(PrototypeVisitor):
             self.emit(s, depth, reflow)
         for argtype, argname, opt in args:
             emit("p->%s = %s;" % (argname, argname), 1)
-        assert not attrs
+        for argtype, argname, opt in attrs:
+            emit("p->%s = %s;" % (argname, argname), 1)
 
 
 class PickleVisitor(EmitVisitor):
@@ -452,10 +455,15 @@ class Obj2ModVisitor(PickleVisitor):
         self.emit("PyObject* tmp = NULL;", 1)
         for f in prod.fields:
             self.visitFieldDeclaration(f, name, prod=prod, depth=1)
+        for a in prod.attributes:
+            self.visitFieldDeclaration(a, name, prod=prod, depth=1)
         self.emit("", 0)
         for f in prod.fields:
             self.visitField(f, name, prod=prod, depth=1)
+        for a in prod.attributes:
+            self.visitField(a, name, prod=prod, depth=1)
         args = [f.name for f in prod.fields]
+        args.extend([a.name for a in prod.attributes])
         self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
         self.emit("return 0;", 1)
         self.emit("failed:", 0)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index a2e9816..07d9b3e 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -2425,7 +2425,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq *
 }
 
 arg_ty
-arg(identifier arg, expr_ty annotation, PyArena *arena)
+arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
+    *arena)
 {
     arg_ty p;
     if (!arg) {
@@ -2438,6 +2439,8 @@ arg(identifier arg, expr_ty annotation, PyArena *arena)
         return NULL;
     p->arg = arg;
     p->annotation = annotation;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
     return p;
 }
 
@@ -7247,6 +7250,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     PyObject* tmp = NULL;
     identifier arg;
     expr_ty annotation;
+    int lineno;
+    int col_offset;
 
     if (_PyObject_HasAttrId(obj, &PyId_arg)) {
         int res;
@@ -7269,7 +7274,29 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     } else {
         annotation = NULL;
     }
-    *out = arg(arg, annotation, arena);
+    if (_PyObject_HasAttrId(obj, &PyId_lineno)) {
+        int res;
+        tmp = _PyObject_GetAttrId(obj, &PyId_lineno);
+        if (tmp == NULL) goto failed;
+        res = obj2ast_int(tmp, &lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg");
+        return 1;
+    }
+    if (_PyObject_HasAttrId(obj, &PyId_col_offset)) {
+        int res;
+        tmp = _PyObject_GetAttrId(obj, &PyId_col_offset);
+        if (tmp == NULL) goto failed;
+        res = obj2ast_int(tmp, &col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg");
+        return 1;
+    }
+    *out = arg(arg, annotation, lineno, col_offset, arena);
     return 0;
 failed:
     Py_XDECREF(tmp);
diff --git a/Python/ast.c b/Python/ast.c
index 5a7a745..77ebc83 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -1181,11 +1181,9 @@ ast_for_arg(struct compiling *c, const node *n)
             return NULL;
     }
 
-    ret = arg(name, annotation, c->c_arena);
+    ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena);
     if (!ret)
         return NULL;
-    ret->lineno = LINENO(n);
-    ret->col_offset = n->n_col_offset;
     return ret;
 }
 
@@ -1241,11 +1239,10 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
                     goto error;
                 if (forbidden_name(c, argname, ch, 0))
                     goto error;
-                arg = arg(argname, annotation, c->c_arena);
+                arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset,
+                          c->c_arena);
                 if (!arg)
                     goto error;
-                arg->lineno = LINENO(ch);
-                arg->col_offset = ch->n_col_offset;
                 asdl_seq_SET(kwonlyargs, j++, arg);
                 i += 2; /* the name and the comma */
                 break;