PEP 448: additional unpacking generalizations (closes #2292)

Patch by Neil Girdhar.

1 files changed, 141 insertions, 124 deletions

diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
index 56ee445..828f46b 100644
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -23,6 +23,11 @@
  *  that lint detects are gone, but there are still warnings with
  *  Py_[X]DECREF() and Py_[X]INCREF() macros.  The lint annotations
  *  look like "NOTE(...)".
+ *
+ *  To debug parser errors like
+ *      "parser.ParserError: Expected node type 12, got 333."
+ *  decode symbol numbers using the automatically-generated files
+ *  Lib/symbol.h and Include/token.h.
  */
 
 #include "Python.h"                     /* general Python API             */
@@ -1087,30 +1092,56 @@ validate_terminal(node *terminal, int type, char *string)
     return (res);
 }
 
+/*  X (',' X) [','] */
+static int
+validate_repeating_list_variable(node *tree,
+                                 int list_node_type,
+                                 int (*validate_child_func_inc)(node *, int *),
+                                 int *pos,
+                                 const char *const list_node_type_name)
+{
+    int nch = NCH(tree);
+    int res = (nch && validate_ntype(tree, list_node_type));
 
-/*  X (',' X) [',']
- */
+    if (!res && !PyErr_Occurred()) {
+        /* Unconditionally raise. */
+        (void) validate_numnodes(tree, 1, list_node_type_name);
+    }
+    else {
+        for ( ; res && *pos < nch; ) {
+            res = validate_child_func_inc(tree, pos);
+            if (!res || *pos >= nch)
+                break;
+            res = validate_comma(CHILD(tree, (*pos)++));
+        }
+    }
+    return res;
+}
+
+/*  X (',' X) [','] */
 static int
-validate_repeating_list(node *tree, int ntype, int (*vfunc)(node *),
-                        const char *const name)
+validate_repeating_list(node *tree,
+                        int list_node_type,
+                        int (*validate_child_func)(node *),
+                        const char *const list_node_type_name)
 {
     int nch = NCH(tree);
-    int res = (nch && validate_ntype(tree, ntype)
-               && vfunc(CHILD(tree, 0)));
+    int res = (nch && validate_ntype(tree, list_node_type));
+    int pos = 0;
 
-    if (!res && !PyErr_Occurred())
-        (void) validate_numnodes(tree, 1, name);
+    if (!res && !PyErr_Occurred()) {
+        /* Unconditionally raise. */
+        (void) validate_numnodes(tree, 1, list_node_type_name);
+    }
     else {
-        if (is_even(nch))
-            res = validate_comma(CHILD(tree, --nch));
-        if (res && nch > 1) {
-            int pos = 1;
-            for ( ; res && pos < nch; pos += 2)
-                res = (validate_comma(CHILD(tree, pos))
-                       && vfunc(CHILD(tree, pos + 1)));
+        for ( ; res && pos < nch; ) {
+            res = validate_child_func(CHILD(tree, pos++));
+            if (!res || pos >= nch)
+                break;
+            res = validate_comma(CHILD(tree, pos++));
         }
     }
-    return (res);
+    return res;
 }
 
 
@@ -2451,9 +2482,9 @@ validate_atom(node *tree)
 
             if (res && (nch == 3)) {
                 if (TYPE(CHILD(tree, 1))==yield_expr)
-                        res = validate_yield_expr(CHILD(tree, 1));
+                    res = validate_yield_expr(CHILD(tree, 1));
                 else
-                        res = validate_testlist_comp(CHILD(tree, 1));
+                    res = validate_testlist_comp(CHILD(tree, 1));
             }
             break;
           case LSQB:
@@ -2493,39 +2524,28 @@ validate_atom(node *tree)
 
 
 /*  testlist_comp:
- *    test ( comp_for | (',' test)* [','] )
+ *    (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
  */
 static int
 validate_testlist_comp(node *tree)
 {
     int nch = NCH(tree);
-    int ok = nch;
+    int ok;
 
-    if (nch == 0)
+    if (nch == 0) {
         err_string("missing child nodes of testlist_comp");
-    else {
-        ok = validate_test_or_star_expr(CHILD(tree, 0));
+        return 0;
     }
 
-    /*
-     *  comp_for | (',' test)* [',']
-     */
-    if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for)
-        ok = validate_comp_for(CHILD(tree, 1));
+    if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for) {
+        ok = (validate_test(CHILD(tree, 0))
+                && validate_comp_for(CHILD(tree, 1)));
+    }
     else {
-        /*  (',' test)* [',']  */
-        int i = 1;
-        while (ok && nch - i >= 2) {
-            ok = (validate_comma(CHILD(tree, i))
-                  && validate_test_or_star_expr(CHILD(tree, i+1)));
-            i += 2;
-        }
-        if (ok && i == nch-1)
-            ok = validate_comma(CHILD(tree, i));
-        else if (i != nch) {
-            ok = 0;
-            err_string("illegal trailing nodes for testlist_comp");
-        }
+        ok = validate_repeating_list(tree,
+                testlist_comp,
+                validate_test_or_star_expr,
+                "testlist_comp");
     }
     return ok;
 }
@@ -2732,9 +2752,6 @@ validate_arglist(node *tree)
     }
     ok = 1;
     if (nch-i > 0) {
-        /*
-         * argument | '*' test [',' '**' test] | '**' test
-         */
         int sym = TYPE(CHILD(tree, i));
 
         if (sym == argument) {
@@ -2745,30 +2762,7 @@ validate_arglist(node *tree)
                 ok = 0;
             }
         }
-        else if (sym == STAR) {
-            ok = validate_star(CHILD(tree, i));
-            if (ok && (nch-i == 2))
-                ok = validate_test(CHILD(tree, i+1));
-            else if (ok && (nch-i == 5))
-                ok = (validate_test(CHILD(tree, i+1))
-                      && validate_comma(CHILD(tree, i+2))
-                      && validate_doublestar(CHILD(tree, i+3))
-                      && validate_test(CHILD(tree, i+4)));
-            else {
-                err_string("illegal use of '*' in arglist");
-                ok = 0;
-            }
-        }
-        else if (sym == DOUBLESTAR) {
-            if (nch-i == 2)
-                ok = (validate_doublestar(CHILD(tree, i))
-                      && validate_test(CHILD(tree, i+1)));
-            else {
-                err_string("illegal use of '**' in arglist");
-                ok = 0;
-            }
-        }
-        else {
+       else {
             err_string("illegal arglist specification");
             ok = 0;
         }
@@ -2778,9 +2772,10 @@ validate_arglist(node *tree)
 
 
 
-/*  argument:
- *
- *  [test '='] test [comp_for]
+/*  argument: ( test [comp_for] |
+ *              test '=' test |
+ *              '**' expr |
+ *              star_expr )
  */
 static int
 validate_argument(node *tree)
@@ -2788,14 +2783,24 @@ validate_argument(node *tree)
     int nch = NCH(tree);
     int res = (validate_ntype(tree, argument)
                && ((nch == 1) || (nch == 2) || (nch == 3)));
-    if (res)
-        res = validate_test(CHILD(tree, 0));
-    if (res && (nch == 2))
-        res = validate_comp_for(CHILD(tree, 1));
-    else if (res && (nch == 3))
-        res = (validate_equal(CHILD(tree, 1))
-               && validate_test(CHILD(tree, 2)));
 
+    if (res) {
+        if (TYPE(CHILD(tree, 0)) == DOUBLESTAR) {
+            res = validate_expr(CHILD(tree, 1));
+        }
+        else if (nch == 1) {
+            res = validate_test_or_star_expr(CHILD(tree, 0));
+        }
+        else if (nch == 2) {
+            res = (validate_test(CHILD(tree, 0))
+                    && validate_comp_for(CHILD(tree, 1)));
+        }
+        else if (res && (nch == 3)) {
+            res = (validate_test(CHILD(tree, 0))
+                    && validate_equal(CHILD(tree, 1))
+                    && validate_test(CHILD(tree, 2)));
+        }
+    }
     return (res);
 }
 
@@ -2948,11 +2953,44 @@ validate_exprlist(node *tree)
                                     validate_expr_or_star_expr, "exprlist"));
 }
 
+/* Incrementing validate functions returns nonzero iff success (like other
+ * validate functions, and advance *i by the length of the matched pattern. */
+
+/* test ':' test */
+static int
+validate_test_colon_test_inc(node *tree, int *i)
+{
+    return (validate_test(CHILD(tree, (*i)++))
+            && validate_colon(CHILD(tree, (*i)++))
+            && validate_test(CHILD(tree, (*i)++)));
+}
+
+/* test ':' test | '**' expr */
+static int
+validate_dict_element_inc(node *tree, int *i)
+{
+    int nch = NCH(tree);
+    int res = 0;
+    if (nch - *i >= 2) {
+        if (TYPE(CHILD(tree, *i+1)) == COLON) {
+            /* test ':' test */
+            res = validate_test_colon_test_inc(tree, i);
+        } else {
+            /* '**' expr */
+            res = (validate_doublestar(CHILD(tree, (*i)++))
+                    && validate_expr(CHILD(tree, (*i)++)));
+        }
+    }
+    return res;
+}
+
 /*
  *  dictorsetmaker:
  *
- *  (test ':' test (comp_for | (',' test ':' test)* [','])) |
- *  (test (comp_for | (',' test)* [',']))
+ *   ( ((test ':' test | '**' expr)
+ *      (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+ *     ((test | '*' test)
+ *      (comp_for | (',' (test | '*' test))* [','])) )
  */
 static int
 validate_dictorsetmaker(node *tree)
@@ -2966,65 +3004,44 @@ validate_dictorsetmaker(node *tree)
         return 0;
 
     if (nch - i < 1) {
+        /* Unconditionally raise. */
         (void) validate_numnodes(tree, 1, "dictorsetmaker");
         return 0;
     }
 
-    res = validate_test(CHILD(tree, i++));
-    if (!res)
-        return 0;
-
-    if (nch - i >= 2 && TYPE(CHILD(tree, i)) == COLON) {
+    if (nch - i >= 2
+        && ((TYPE(CHILD(tree, i+1)) == COLON) ||
+            (TYPE(CHILD(tree, i)) == DOUBLESTAR))) {
         /* Dictionary display or dictionary comprehension. */
-        res = (validate_colon(CHILD(tree, i++))
-               && validate_test(CHILD(tree, i++)));
-        if (!res)
-            return 0;
-
-        if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) {
+        if (nch - i >= 4 && TYPE(CHILD(tree, i+3)) == comp_for) {
             /* Dictionary comprehension. */
-            res = validate_comp_for(CHILD(tree, i++));
+            res = (validate_test_colon_test_inc(tree, &i)
+                    && validate_comp_for(CHILD(tree, i++)));
             if (!res)
                 return 0;
-        }
-        else {
+        } else {
             /* Dictionary display. */
-            while (nch - i >= 4) {
-                res = (validate_comma(CHILD(tree, i++))
-                       && validate_test(CHILD(tree, i++))
-                       && validate_colon(CHILD(tree, i++))
-                       && validate_test(CHILD(tree, i++)));
-                if (!res)
-                    return 0;
-            }
-            if (nch - i == 1) {
-                res = validate_comma(CHILD(tree, i++));
-                if (!res)
-                    return 0;
-            }
+            return validate_repeating_list_variable(
+                    tree,
+                    dictorsetmaker,
+                    validate_dict_element_inc,
+                    &i,
+                    "dictorsetmaker");
         }
-    }
-    else {
+    } else {
         /* Set display or set comprehension. */
-        if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) {
+        if (nch - i >= 2 && TYPE(CHILD(tree, i + 1)) == comp_for) {
             /* Set comprehension. */
-            res = validate_comp_for(CHILD(tree, i++));
+            res = (validate_test(CHILD(tree, i++))
+                   && validate_comp_for(CHILD(tree, i++)));
             if (!res)
                 return 0;
-        }
-        else {
+        } else {
             /* Set display. */
-            while (nch - i >= 2) {
-                res = (validate_comma(CHILD(tree, i++))
-                       && validate_test(CHILD(tree, i++)));
-                if (!res)
-                    return 0;
-            }
-            if (nch - i == 1) {
-                res = validate_comma(CHILD(tree, i++));
-                if (!res)
-                    return 0;
-            }
+           return validate_repeating_list(tree,
+                                          dictorsetmaker,
+                                          validate_test_or_star_expr,
+                                          "dictorsetmaker");
         }
     }