unify some ast.argument's attrs; change Attribute column offset (closes #16795)

Patch from Sven Brauch.

3 files changed, 49 insertions, 8 deletions

diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 0d373b0..debd89e 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -103,11 +103,11 @@ module Python
     excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
                     attributes (int lineno, int col_offset)
 
-    arguments = (arg* args, identifier? vararg, expr? varargannotation,
-                     arg* kwonlyargs, identifier? kwarg,
-                     expr? kwargannotation, expr* defaults,
-                     expr* kw_defaults)
+    arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
+                 arg? kwarg, expr* defaults)
+
     arg = (identifier arg, expr? annotation)
+           attributes (int lineno, int col_offset)
 
     -- keyword arguments supplied to call
     keyword = (identifier arg, expr value)
diff --git a/Parser/asdl.py b/Parser/asdl.py
index 1f98ada..7df76c0 100644
--- a/Parser/asdl.py
+++ b/Parser/asdl.py
@@ -158,11 +158,19 @@ class ASDLParser(spark.GenericParser, object):
                                   msg="expected attributes, found %s" % id)
         return Sum(sum, attributes)
 
-    def p_product(self, info):
+    def p_product_0(self, info):
         " product ::= ( fields ) "
         _0, fields, _1 = info
         return Product(fields)
 
+    def p_product_1(self, info):
+        " product ::= ( fields ) Id ( fields ) "
+        _0, fields, _1, id, _2, attributes, _3 = info
+        if id.value != "attributes":
+            raise ASDLSyntaxError(id.lineno,
+                                  msg="expected attributes, found %s" % id)
+        return Product(fields, attributes)
+
     def p_sum_0(self, constructor):
         " sum ::= constructor "
         return [constructor[0]]
@@ -289,11 +297,15 @@ class Sum(AST):
             return "Sum(%s, %s)" % (self.types, self.attributes)
 
 class Product(AST):
-    def __init__(self, fields):
+    def __init__(self, fields, attributes=None):
         self.fields = fields
+        self.attributes = attributes or []
 
     def __repr__(self):
-        return "Product(%s)" % self.fields
+        if self.attributes is None:
+            return "Product(%s)" % self.fields
+        else:
+            return "Product(%s, %s)" % (self.fields, self.attributes)
 
 class VisitorBase(object):
 
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 9557ba0..61747ed 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -209,6 +209,11 @@ class StructVisitor(EmitVisitor):
         self.emit("struct _%(name)s {" % locals(), depth)
         for f in product.fields:
             self.visit(f, depth + 1)
+        for field in product.attributes:
+            # rudimentary attribute handling
+            type = str(field.type)
+            assert type in asdl.builtin_types, type
+            self.emit("%s %s;" % (type, field.name), depth + 1);
         self.emit("};", depth)
         self.emit("", depth)
 
@@ -493,7 +498,13 @@ class Obj2ModVisitor(PickleVisitor):
 
     def visitField(self, field, name, sum=None, prod=None, depth=0):
         ctype = get_c_type(field.type)
-        self.emit("if (_PyObject_HasAttrId(obj, &PyId_%s)) {" % field.name, depth)
+        if field.opt:
+            add_check = " && _PyObject_GetAttrId(obj, &PyId_%s) != Py_None" \
+                        % (field.name)
+        else:
+            add_check = str()
+        self.emit("if (_PyObject_HasAttrId(obj, &PyId_%s)%s) {"
+                   % (field.name, add_check), depth, reflow=False)
         self.emit("int res;", depth+1)
         if field.seq:
             self.emit("Py_ssize_t len;", depth+1)
@@ -559,6 +570,13 @@ class PyTypesDeclareVisitor(PickleVisitor):
     def visitProduct(self, prod, name):
         self.emit("static PyTypeObject *%s_type;" % name, 0)
         self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
+        if prod.attributes:
+            for a in prod.attributes:
+                self.emit_identifier(a.name)
+            self.emit("static char *%s_attributes[] = {" % name, 0)
+            for a in prod.attributes:
+                self.emit('"%s",' % a.name, 1)
+            self.emit("};", 0)
         if prod.fields:
             for f in prod.fields:
                 self.emit_identifier(f.name)
@@ -934,6 +952,11 @@ static int add_ast_fields(void)
         self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
                         (name, name, fields, len(prod.fields)), 1)
         self.emit("if (!%s_type) return 0;" % name, 1)
+        if prod.attributes:
+            self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
+                            (name, name, len(prod.attributes)), 1)
+        else:
+            self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
 
     def visitSum(self, sum, name):
         self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
@@ -1089,6 +1112,12 @@ class ObjVisitor(PickleVisitor):
         self.emit("if (!result) return NULL;", 1)
         for field in prod.fields:
             self.visitField(field, name, 1, True)
+        for a in prod.attributes:
+            self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
+            self.emit("if (!value) goto failed;", 1)
+            self.emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) < 0)' % a.name, 1)
+            self.emit('goto failed;', 2)
+            self.emit('Py_DECREF(value);', 1)
         self.func_end()
 
     def visitConstructor(self, cons, enum, name):