make YieldFrom its own distinct from Yield (closes #13780)

7 files changed, 84 insertions, 32 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 4e21674e..7ad6cb3 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -180,10 +180,10 @@ struct _stmt {
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
-                  Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15,
-                  Str_kind=16, Bytes_kind=17, Ellipsis_kind=18,
-                  Attribute_kind=19, Subscript_kind=20, Starred_kind=21,
-                  Name_kind=22, List_kind=23, Tuple_kind=24};
+                  Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
+                  Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
+                  Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
+                  Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
 struct _expr {
         enum _expr_kind kind;
         union {
@@ -245,11 +245,14 @@ struct _expr {
                 } GeneratorExp;
                 
                 struct {
-                        int is_from;
                         expr_ty value;
                 } Yield;
                 
                 struct {
+                        expr_ty value;
+                } YieldFrom;
+                
+                struct {
                         expr_ty left;
                         asdl_int_seq *ops;
                         asdl_seq *comparators;
@@ -488,9 +491,11 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
 #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
                          col_offset, PyArena *arena);
-#define Yield(a0, a1, a2, a3, a4) _Py_Yield(a0, a1, a2, a3, a4)
-expr_ty _Py_Yield(int is_from, expr_ty value, int lineno, int col_offset,
-                  PyArena *arena);
+#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
+expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
+#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3)
+expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
+                      *arena);
 #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
 expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
                     int lineno, int col_offset, PyArena *arena);
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 2022ee2..b809d24 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -813,8 +813,8 @@ class ASTValidatorTests(unittest.TestCase):
         self._check_comprehension(factory)
 
     def test_yield(self):
-        self.expr(ast.Yield(0, ast.Name("x", ast.Store())), "must have Load")
-        self.expr(ast.Yield(1, ast.Name("x", ast.Store())), "must have Load")
+        self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load")
+        self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load")
 
     def test_compare(self):
         left = ast.Name("x", ast.Load())
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 8ae4e59..876c175 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -59,7 +59,8 @@ module Python
 	     | DictComp(expr key, expr value, comprehension* generators)
 	     | GeneratorExp(expr elt, comprehension* generators)
 	     -- the grammar constrains where yield expressions can occur
-	     | Yield(int is_from, expr? value)
+	     | Yield(expr? value)
+             | YieldFrom(expr? value)
 	     -- need sequences for compare to distinguish between
 	     -- x < 4 < 3 and (x < 4) < 3
 	     | Compare(expr left, cmpop* ops, expr* comparators)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 2882bbc..2603b81 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -231,9 +231,11 @@ static char *GeneratorExp_fields[]={
         "generators",
 };
 static PyTypeObject *Yield_type;
-_Py_IDENTIFIER(is_from);
 static char *Yield_fields[]={
-        "is_from",
+        "value",
+};
+static PyTypeObject *YieldFrom_type;
+static char *YieldFrom_fields[]={
         "value",
 };
 static PyTypeObject *Compare_type;
@@ -812,8 +814,10 @@ static int init_types(void)
         GeneratorExp_type = make_type("GeneratorExp", expr_type,
                                       GeneratorExp_fields, 2);
         if (!GeneratorExp_type) return 0;
-        Yield_type = make_type("Yield", expr_type, Yield_fields, 2);
+        Yield_type = make_type("Yield", expr_type, Yield_fields, 1);
         if (!Yield_type) return 0;
+        YieldFrom_type = make_type("YieldFrom", expr_type, YieldFrom_fields, 1);
+        if (!YieldFrom_type) return 0;
         Compare_type = make_type("Compare", expr_type, Compare_fields, 3);
         if (!Compare_type) return 0;
         Call_type = make_type("Call", expr_type, Call_fields, 5);
@@ -1749,14 +1753,13 @@ GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset,
 }
 
 expr_ty
-Yield(int is_from, expr_ty value, int lineno, int col_offset, PyArena *arena)
+Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
         expr_ty p;
         p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
         if (!p)
                 return NULL;
         p->kind = Yield_kind;
-        p->v.Yield.is_from = is_from;
         p->v.Yield.value = value;
         p->lineno = lineno;
         p->col_offset = col_offset;
@@ -1764,6 +1767,20 @@ Yield(int is_from, expr_ty value, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
+YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
+{
+        expr_ty p;
+        p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+        if (!p)
+                return NULL;
+        p->kind = YieldFrom_kind;
+        p->v.YieldFrom.value = value;
+        p->lineno = lineno;
+        p->col_offset = col_offset;
+        return p;
+}
+
+expr_ty
 Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
         int col_offset, PyArena *arena)
 {
@@ -2798,12 +2815,16 @@ ast2obj_expr(void* _o)
         case Yield_kind:
                 result = PyType_GenericNew(Yield_type, NULL, NULL);
                 if (!result) goto failed;
-                value = ast2obj_int(o->v.Yield.is_from);
+                value = ast2obj_expr(o->v.Yield.value);
                 if (!value) goto failed;
-                if (_PyObject_SetAttrId(result, &PyId_is_from, value) == -1)
+                if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
                         goto failed;
                 Py_DECREF(value);
-                value = ast2obj_expr(o->v.Yield.value);
+                break;
+        case YieldFrom_kind:
+                result = PyType_GenericNew(YieldFrom_type, NULL, NULL);
+                if (!result) goto failed;
+                value = ast2obj_expr(o->v.YieldFrom.value);
                 if (!value) goto failed;
                 if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
                         goto failed;
@@ -5345,21 +5366,30 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                 return 1;
         }
         if (isinstance) {
-                int is_from;
                 expr_ty value;
 
-                if (_PyObject_HasAttrId(obj, &PyId_is_from)) {
+                if (_PyObject_HasAttrId(obj, &PyId_value)) {
                         int res;
-                        tmp = _PyObject_GetAttrId(obj, &PyId_is_from);
+                        tmp = _PyObject_GetAttrId(obj, &PyId_value);
                         if (tmp == NULL) goto failed;
-                        res = obj2ast_int(tmp, &is_from, arena);
+                        res = obj2ast_expr(tmp, &value, arena);
                         if (res != 0) goto failed;
                         Py_XDECREF(tmp);
                         tmp = NULL;
                 } else {
-                        PyErr_SetString(PyExc_TypeError, "required field \"is_from\" missing from Yield");
-                        return 1;
+                        value = NULL;
                 }
+                *out = Yield(value, lineno, col_offset, arena);
+                if (*out == NULL) goto failed;
+                return 0;
+        }
+        isinstance = PyObject_IsInstance(obj, (PyObject*)YieldFrom_type);
+        if (isinstance == -1) {
+                return 1;
+        }
+        if (isinstance) {
+                expr_ty value;
+
                 if (_PyObject_HasAttrId(obj, &PyId_value)) {
                         int res;
                         tmp = _PyObject_GetAttrId(obj, &PyId_value);
@@ -5371,7 +5401,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
                 } else {
                         value = NULL;
                 }
-                *out = Yield(is_from, value, lineno, col_offset, arena);
+                *out = YieldFrom(value, lineno, col_offset, arena);
                 if (*out == NULL) goto failed;
                 return 0;
         }
@@ -6928,6 +6958,8 @@ PyInit__ast(void)
             (PyObject*)GeneratorExp_type) < 0) return NULL;
         if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return
             NULL;
+        if (PyDict_SetItemString(d, "YieldFrom", (PyObject*)YieldFrom_type) <
+            0) return NULL;
         if (PyDict_SetItemString(d, "Compare", (PyObject*)Compare_type) < 0)
             return NULL;
         if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return
diff --git a/Python/ast.c b/Python/ast.c
index 7080c65..3417fe3 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -223,6 +223,9 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
             validate_expr(exp->v.DictComp.value, Load);
     case Yield_kind:
         return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load);
+    case YieldFrom_kind:
+        return !exp->v.YieldFrom.value ||
+            validate_expr(exp->v.YieldFrom.value, Load);
     case Compare_kind:
         if (!asdl_seq_LEN(exp->v.Compare.comparators)) {
             PyErr_SetString(PyExc_ValueError, "Compare with no comparators");
@@ -942,6 +945,7 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
             expr_name = "generator expression";
             break;
         case Yield_kind:
+        case YieldFrom_kind:
             expr_name = "yield expression";
             break;
         case ListComp_kind:
@@ -2386,7 +2390,9 @@ ast_for_expr(struct compiling *c, const node *n)
                 if (!exp)
                     return NULL;
             }
-            return Yield(is_from, exp, LINENO(n), n->n_col_offset, c->c_arena);
+            if (is_from)
+                return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena);
+            return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
         }
         case factor:
             if (NCH(n) == 1) {
diff --git a/Python/compile.c b/Python/compile.c
index 4d91f50..b64c800 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3311,21 +3311,25 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case DictComp_kind:
         return compiler_dictcomp(c, e);
     case Yield_kind:
+    case YieldFrom_kind: {
+        expr_ty value;
         if (c->u->u_ste->ste_type != FunctionBlock)
             return compiler_error(c, "'yield' outside function");
-        if (e->v.Yield.value) {
-            VISIT(c, expr, e->v.Yield.value);
+        value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value;
+        if (value) {
+            VISIT(c, expr, value);
         }
         else {
             ADDOP_O(c, LOAD_CONST, Py_None, consts);
         }
-        if (e->v.Yield.is_from) {
+        if (e->kind == YieldFrom_kind) {
             ADDOP(c, YIELD_FROM);
         }
         else {
             ADDOP(c, YIELD_VALUE);
         }
         break;
+    }
     case Compare_kind:
         return compiler_compare(c, e);
     case Call_kind:
diff --git a/Python/symtable.c b/Python/symtable.c
index 1ce7f70..b622835 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1330,10 +1330,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
             return 0;
         break;
     case Yield_kind:
-        if (e->v.Yield.value)
-            VISIT(st, expr, e->v.Yield.value);
+    case YieldFrom_kind: {
+        expr_ty value;
+        value = (e->kind == YieldFrom_kind) ? e->v.YieldFrom.value : e->v.Yield.value;
+        if (value)
+            VISIT(st, expr, value);
         st->st_cur->ste_generator = 1;
         break;
+    }
     case Compare_kind:
         VISIT(st, expr, e->v.Compare.left);
         VISIT_SEQ(st, expr, e->v.Compare.comparators);