perform yield from delegation by repeating YIELD_FROM opcode (closes #14230)

This allows generators that are using yield from to be seen by debuggers. It
also kills the f_yieldfrom field on frame objects.

Patch mostly from Mark Shannon with a few tweaks by me.

3 files changed, 29 insertions, 38 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 017dc4a..5498056 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1170,6 +1170,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
        f->f_lasti to -1 (i.e. the index *before* the first instruction)
        and YIELD_VALUE doesn't fiddle with f_lasti any more.  So this
        does work.  Promise.
+       YIELD_FROM sets f_lasti to itself, in order to repeated yield
+       multiple values.
 
        When the PREDICT() macros are enabled, some opcode pairs follow in
        direct succession without updating f->f_lasti.  A successful
@@ -1830,49 +1832,35 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
         TARGET(YIELD_FROM)
             u = POP();
-            x = PyObject_GetIter(u);
+            x = TOP();
+            /* send u to x */
+            if (PyGen_CheckExact(x)) {
+                retval = _PyGen_Send((PyGenObject *)x, u);
+            } else {
+                if (u == Py_None)
+                    retval = PyIter_Next(x);
+                else
+                    retval = PyObject_CallMethod(x, "send", "O", u);
+            }
             Py_DECREF(u);
-            if (x == NULL)
-                break;
-            /* x is now the iterator, make the first next() call */
-            retval = (*Py_TYPE(x)->tp_iternext)(x);
             if (!retval) {
-                PyObject *et, *ev, *tb;
-                /* iter may be exhausted */
-                Py_CLEAR(x);
-                if (PyErr_Occurred() &&
-                    !PyErr_ExceptionMatches(PyExc_StopIteration)) {
-                    /* some other exception */
+                PyObject *val;
+                x = POP(); /* Remove iter from stack */
+                Py_DECREF(x);
+                err = PyGen_FetchStopIterationValue(&val);
+                if (err < 0) {
+                    x = NULL;
                     break;
                 }
-                /* try to get return value from exception */
-                PyErr_Fetch(&et, &ev, &tb);
-                Py_XDECREF(et);
-                Py_XDECREF(tb);
-                /* u is return value */
-                u = NULL;
-                if (ev) {
-                    u = PyObject_GetAttrString(ev, "value");
-                    Py_DECREF(ev);
-                    if (u == NULL) {
-                        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                            /* some other exception */
-                            break;
-                        }
-                        PyErr_Clear();
-                    }
-                }
-                if (u == NULL) {
-                    u = Py_None;
-                    Py_INCREF(u);
-                }
-                PUSH(u);
+                x = val;
+                PUSH(x);
                 continue;
             }
-            /* x is iterator, retval is value to be yielded */
-            f->f_yieldfrom = x;
+            /* x remains on stack, retval is value to be yielded */
             f->f_stacktop = stack_pointer;
             why = WHY_YIELD;
+            /* and repeat... */
+            f->f_lasti--;
             goto fast_yield;
 
         TARGET(YIELD_VALUE)
diff --git a/Python/compile.c b/Python/compile.c
index b64c800..1722a5f 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -840,9 +840,9 @@ opcode_stack_effect(int opcode, int oparg)
         case IMPORT_STAR:
             return -1;
         case YIELD_VALUE:
-        case YIELD_FROM:
             return 0;
-
+        case YIELD_FROM:
+            return -1;
         case POP_BLOCK:
             return 0;
         case POP_EXCEPT:
@@ -3323,6 +3323,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
             ADDOP_O(c, LOAD_CONST, Py_None, consts);
         }
         if (e->kind == YieldFrom_kind) {
+            ADDOP(c, GET_ITER);
+            ADDOP_O(c, LOAD_CONST, Py_None, consts);
             ADDOP(c, YIELD_FROM);
         }
         else {
diff --git a/Python/import.c b/Python/import.c
index 4871b99..14148c7 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -104,7 +104,8 @@ typedef unsigned short mode_t;
        Python 3.2a2  3180 (add DELETE_DEREF)
        Python 3.3a0  3190 __class__ super closure changed
        Python 3.3a0  3200 (__qualname__ added)
-                     3210 (added size modulo 2**32 to the pyc header)
+       Python 3.3a1  3210 (added size modulo 2**32 to the pyc header)
+                     3220 (changed PEP 380 implementation)
 */
 
 /* MAGIC must change whenever the bytecode emitted by the compiler may no