Very subtle syntax change: in a list comprehension, the testlist in

"for <var> in <testlist> may no longer be a single test followed by
a comma.  This solves SF bug #431886.  Note that if the testlist
contains more than one test, a trailing comma is still allowed, for
maximum backward compatibility; but this example is not:

    [(x, y) for x in range(10), for y in range(10)]
                              ^

The fix involved creating a new nonterminal 'testlist_safe' whose
definition doesn't allow the trailing comma if there's only one test:

    testlist_safe: test [(',' test)+ [',']]

5 files changed, 150 insertions, 111 deletions

diff --git a/Grammar/Grammar b/Grammar/Grammar
index d85863e..6ac0cbc 100644
--- a/Grammar/Grammar
+++ b/Grammar/Grammar
@@ -89,6 +89,7 @@ subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: expr (',' expr)* [',']
 testlist: test (',' test)* [',']
+testlist_safe: test [(',' test)+ [',']]
 dictmaker: test ':' test (',' test ':' test)* [',']
 
 classdef: 'class' NAME ['(' testlist ')'] ':' suite
@@ -97,5 +98,5 @@ arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test)
 argument: [test '='] test	# Really [keyword '='] test
 
 list_iter: list_for | list_if
-list_for: 'for' exprlist 'in' testlist [list_iter]
+list_for: 'for' exprlist 'in' testlist_safe [list_iter]
 list_if: 'if' test [list_iter]
diff --git a/Include/graminit.h b/Include/graminit.h
index 219be94..e3afb93 100644
--- a/Include/graminit.h
+++ b/Include/graminit.h
@@ -56,10 +56,11 @@
 #define sliceop 311
 #define exprlist 312
 #define testlist 313
-#define dictmaker 314
-#define classdef 315
-#define arglist 316
-#define argument 317
-#define list_iter 318
-#define list_for 319
-#define list_if 320
+#define testlist_safe 314
+#define dictmaker 315
+#define classdef 316
+#define arglist 317
+#define argument 318
+#define list_iter 319
+#define list_for 320
+#define list_if 321
diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c
index bc82ce5..000e639 100644
--- a/Modules/parsermodule.c
+++ b/Modules/parsermodule.c
@@ -1039,6 +1039,14 @@ validate_testlist(node *tree)
 }
 
 
+static int
+validate_testlist_safe(node *tree)
+{
+    return (validate_repeating_list(tree, testlist_safe,
+                                    validate_test, "testlist_safe"));
+}
+
+
 /* '*' NAME [',' '**' NAME] | '**' NAME
  */
 static int
@@ -1218,7 +1226,7 @@ validate_list_for(node *tree)
         res = (validate_name(CHILD(tree, 0), "for")
                && validate_exprlist(CHILD(tree, 1))
                && validate_name(CHILD(tree, 2), "in")
-               && validate_testlist(CHILD(tree, 3)));
+               && validate_testlist_safe(CHILD(tree, 3)));
 
     return res;
 }
diff --git a/Python/compile.c b/Python/compile.c
index dae2a3e..5120916 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3710,6 +3710,7 @@ com_node(struct compiling *c, node *n)
 	/* Expression nodes */
 	
 	case testlist:
+	case testlist_safe:
 		com_list(c, n, 0);
 		break;
 	case test:
diff --git a/Python/graminit.c b/Python/graminit.c
index 176f62d..11e0705 100644
--- a/Python/graminit.c
+++ b/Python/graminit.c
@@ -1255,8 +1255,9 @@ static state states_57[3] = {
 static arc arcs_58_0[1] = {
 	{21, 1},
 };
-static arc arcs_58_1[1] = {
-	{14, 2},
+static arc arcs_58_1[2] = {
+	{22, 2},
+	{0, 1},
 };
 static arc arcs_58_2[1] = {
 	{21, 3},
@@ -1266,173 +1267,197 @@ static arc arcs_58_3[2] = {
 	{0, 3},
 };
 static arc arcs_58_4[2] = {
-	{21, 1},
+	{21, 3},
 	{0, 4},
 };
 static state states_58[5] = {
 	{1, arcs_58_0},
-	{1, arcs_58_1},
+	{2, arcs_58_1},
 	{1, arcs_58_2},
 	{2, arcs_58_3},
 	{2, arcs_58_4},
 };
 static arc arcs_59_0[1] = {
-	{142, 1},
+	{21, 1},
 };
 static arc arcs_59_1[1] = {
+	{14, 2},
+};
+static arc arcs_59_2[1] = {
+	{21, 3},
+};
+static arc arcs_59_3[2] = {
+	{22, 4},
+	{0, 3},
+};
+static arc arcs_59_4[2] = {
+	{21, 1},
+	{0, 4},
+};
+static state states_59[5] = {
+	{1, arcs_59_0},
+	{1, arcs_59_1},
+	{1, arcs_59_2},
+	{2, arcs_59_3},
+	{2, arcs_59_4},
+};
+static arc arcs_60_0[1] = {
+	{143, 1},
+};
+static arc arcs_60_1[1] = {
 	{12, 2},
 };
-static arc arcs_59_2[2] = {
+static arc arcs_60_2[2] = {
 	{16, 3},
 	{14, 4},
 };
-static arc arcs_59_3[1] = {
+static arc arcs_60_3[1] = {
 	{9, 5},
 };
-static arc arcs_59_4[1] = {
+static arc arcs_60_4[1] = {
 	{15, 6},
 };
-static arc arcs_59_5[1] = {
+static arc arcs_60_5[1] = {
 	{18, 7},
 };
-static arc arcs_59_6[1] = {
+static arc arcs_60_6[1] = {
 	{0, 6},
 };
-static arc arcs_59_7[1] = {
+static arc arcs_60_7[1] = {
 	{14, 4},
 };
-static state states_59[8] = {
-	{1, arcs_59_0},
-	{1, arcs_59_1},
-	{2, arcs_59_2},
-	{1, arcs_59_3},
-	{1, arcs_59_4},
-	{1, arcs_59_5},
-	{1, arcs_59_6},
-	{1, arcs_59_7},
-};
-static arc arcs_60_0[3] = {
-	{143, 1},
+static state states_60[8] = {
+	{1, arcs_60_0},
+	{1, arcs_60_1},
+	{2, arcs_60_2},
+	{1, arcs_60_3},
+	{1, arcs_60_4},
+	{1, arcs_60_5},
+	{1, arcs_60_6},
+	{1, arcs_60_7},
+};
+static arc arcs_61_0[3] = {
+	{144, 1},
 	{23, 2},
 	{24, 3},
 };
-static arc arcs_60_1[2] = {
+static arc arcs_61_1[2] = {
 	{22, 4},
 	{0, 1},
 };
-static arc arcs_60_2[1] = {
+static arc arcs_61_2[1] = {
 	{21, 5},
 };
-static arc arcs_60_3[1] = {
+static arc arcs_61_3[1] = {
 	{21, 6},
 };
-static arc arcs_60_4[4] = {
-	{143, 1},
+static arc arcs_61_4[4] = {
+	{144, 1},
 	{23, 2},
 	{24, 3},
 	{0, 4},
 };
-static arc arcs_60_5[2] = {
+static arc arcs_61_5[2] = {
 	{22, 7},
 	{0, 5},
 };
-static arc arcs_60_6[1] = {
+static arc arcs_61_6[1] = {
 	{0, 6},
 };
-static arc arcs_60_7[1] = {
+static arc arcs_61_7[1] = {
 	{24, 3},
 };
-static state states_60[8] = {
-	{3, arcs_60_0},
-	{2, arcs_60_1},
-	{1, arcs_60_2},
-	{1, arcs_60_3},
-	{4, arcs_60_4},
-	{2, arcs_60_5},
-	{1, arcs_60_6},
-	{1, arcs_60_7},
+static state states_61[8] = {
+	{3, arcs_61_0},
+	{2, arcs_61_1},
+	{1, arcs_61_2},
+	{1, arcs_61_3},
+	{4, arcs_61_4},
+	{2, arcs_61_5},
+	{1, arcs_61_6},
+	{1, arcs_61_7},
 };
-static arc arcs_61_0[1] = {
+static arc arcs_62_0[1] = {
 	{21, 1},
 };
-static arc arcs_61_1[2] = {
+static arc arcs_62_1[2] = {
 	{20, 2},
 	{0, 1},
 };
-static arc arcs_61_2[1] = {
+static arc arcs_62_2[1] = {
 	{21, 3},
 };
-static arc arcs_61_3[1] = {
+static arc arcs_62_3[1] = {
 	{0, 3},
 };
-static state states_61[4] = {
-	{1, arcs_61_0},
-	{2, arcs_61_1},
-	{1, arcs_61_2},
-	{1, arcs_61_3},
+static state states_62[4] = {
+	{1, arcs_62_0},
+	{2, arcs_62_1},
+	{1, arcs_62_2},
+	{1, arcs_62_3},
 };
-static arc arcs_62_0[2] = {
+static arc arcs_63_0[2] = {
 	{136, 1},
-	{145, 1},
+	{146, 1},
 };
-static arc arcs_62_1[1] = {
+static arc arcs_63_1[1] = {
 	{0, 1},
 };
-static state states_62[2] = {
-	{2, arcs_62_0},
-	{1, arcs_62_1},
+static state states_63[2] = {
+	{2, arcs_63_0},
+	{1, arcs_63_1},
 };
-static arc arcs_63_0[1] = {
+static arc arcs_64_0[1] = {
 	{85, 1},
 };
-static arc arcs_63_1[1] = {
+static arc arcs_64_1[1] = {
 	{53, 2},
 };
-static arc arcs_63_2[1] = {
+static arc arcs_64_2[1] = {
 	{74, 3},
 };
-static arc arcs_63_3[1] = {
-	{9, 4},
+static arc arcs_64_3[1] = {
+	{142, 4},
 };
-static arc arcs_63_4[2] = {
-	{144, 5},
+static arc arcs_64_4[2] = {
+	{145, 5},
 	{0, 4},
 };
-static arc arcs_63_5[1] = {
+static arc arcs_64_5[1] = {
 	{0, 5},
 };
-static state states_63[6] = {
-	{1, arcs_63_0},
-	{1, arcs_63_1},
-	{1, arcs_63_2},
-	{1, arcs_63_3},
-	{2, arcs_63_4},
-	{1, arcs_63_5},
+static state states_64[6] = {
+	{1, arcs_64_0},
+	{1, arcs_64_1},
+	{1, arcs_64_2},
+	{1, arcs_64_3},
+	{2, arcs_64_4},
+	{1, arcs_64_5},
 };
-static arc arcs_64_0[1] = {
+static arc arcs_65_0[1] = {
 	{81, 1},
 };
-static arc arcs_64_1[1] = {
+static arc arcs_65_1[1] = {
 	{21, 2},
 };
-static arc arcs_64_2[2] = {
-	{144, 3},
+static arc arcs_65_2[2] = {
+	{145, 3},
 	{0, 2},
 };
-static arc arcs_64_3[1] = {
+static arc arcs_65_3[1] = {
 	{0, 3},
 };
-static state states_64[4] = {
-	{1, arcs_64_0},
-	{1, arcs_64_1},
-	{2, arcs_64_2},
-	{1, arcs_64_3},
+static state states_65[4] = {
+	{1, arcs_65_0},
+	{1, arcs_65_1},
+	{2, arcs_65_2},
+	{1, arcs_65_3},
 };
-static dfa dfas[65] = {
+static dfa dfas[66] = {
 	{256, "single_input", 0, 3, states_0,
-	 "\004\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\102\000"},
+	 "\004\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\202\000"},
 	{257, "file_input", 0, 2, states_1,
-	 "\204\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\102\000"},
+	 "\204\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\202\000"},
 	{258, "eval_input", 0, 3, states_2,
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
 	{259, "funcdef", 0, 6, states_3,
@@ -1446,7 +1471,7 @@ static dfa dfas[65] = {
 	{263, "fplist", 0, 3, states_7,
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"},
 	{264, "stmt", 0, 2, states_8,
-	 "\000\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\102\000"},
+	 "\000\030\001\000\000\000\124\360\213\011\162\000\002\000\140\210\344\202\000"},
 	{265, "simple_stmt", 0, 4, states_9,
 	 "\000\020\001\000\000\000\124\360\213\011\000\000\002\000\140\210\344\002\000"},
 	{266, "small_stmt", 0, 2, states_10,
@@ -1488,7 +1513,7 @@ static dfa dfas[65] = {
 	{284, "assert_stmt", 0, 5, states_28,
 	 "\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000"},
 	{285, "compound_stmt", 0, 2, states_29,
-	 "\000\010\000\000\000\000\000\000\000\000\162\000\000\000\000\000\000\100\000"},
+	 "\000\010\000\000\000\000\000\000\000\000\162\000\000\000\000\000\000\200\000"},
 	{286, "if_stmt", 0, 8, states_30,
 	 "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000"},
 	{287, "while_stmt", 0, 8, states_31,
@@ -1545,22 +1570,24 @@ static dfa dfas[65] = {
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\000\000\140\210\344\000\000"},
 	{313, "testlist", 0, 3, states_57,
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
-	{314, "dictmaker", 0, 5, states_58,
+	{314, "testlist_safe", 0, 5, states_58,
+	 "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
+	{315, "dictmaker", 0, 5, states_59,
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
-	{315, "classdef", 0, 8, states_59,
-	 "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\100\000"},
-	{316, "arglist", 0, 8, states_60,
+	{316, "classdef", 0, 8, states_60,
+	 "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"},
+	{317, "arglist", 0, 8, states_61,
 	 "\000\020\201\001\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
-	{317, "argument", 0, 4, states_61,
+	{318, "argument", 0, 4, states_62,
 	 "\000\020\001\000\000\000\000\000\000\000\000\000\002\000\140\210\344\002\000"},
-	{318, "list_iter", 0, 2, states_62,
+	{319, "list_iter", 0, 2, states_63,
 	 "\000\000\000\000\000\000\000\000\000\000\042\000\000\000\000\000\000\000\000"},
-	{319, "list_for", 0, 6, states_63,
+	{320, "list_for", 0, 6, states_64,
 	 "\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000"},
-	{320, "list_if", 0, 4, states_64,
+	{321, "list_if", 0, 4, states_65,
 	 "\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000\000"},
 };
-static label labels[146] = {
+static label labels[147] = {
 	{0, "EMPTY"},
 	{256, 0},
 	{4, 0},
@@ -1641,7 +1668,7 @@ static label labels[146] = {
 	{287, 0},
 	{288, 0},
 	{289, 0},
-	{315, 0},
+	{316, 0},
 	{1, "if"},
 	{1, "elif"},
 	{1, "else"},
@@ -1692,25 +1719,26 @@ static label labels[146] = {
 	{306, 0},
 	{10, 0},
 	{26, 0},
-	{314, 0},
+	{315, 0},
 	{27, 0},
 	{25, 0},
 	{2, 0},
 	{3, 0},
-	{319, 0},
+	{320, 0},
 	{1, "lambda"},
-	{316, 0},
+	{317, 0},
 	{309, 0},
 	{310, 0},
 	{311, 0},
+	{314, 0},
 	{1, "class"},
-	{317, 0},
 	{318, 0},
-	{320, 0},
+	{319, 0},
+	{321, 0},
 };
 grammar _PyParser_Grammar = {
-	65,
+	66,
 	dfas,
-	{146, labels},
+	{147, labels},
 	256
 };