diff options
| -rw-r--r-- | Include/frameobject.h | 1 | ||||
| -rw-r--r-- | Include/genobject.h | 3 | ||||
| -rw-r--r-- | Lib/test/test_pep380.py | 23 | ||||
| -rw-r--r-- | Lib/test/test_sys.py | 2 | ||||
| -rw-r--r-- | Objects/frameobject.c | 4 | ||||
| -rw-r--r-- | Objects/genobject.c | 179 | ||||
| -rw-r--r-- | Python/ceval.c | 58 | ||||
| -rw-r--r-- | Python/compile.c | 6 | ||||
| -rw-r--r-- | Python/import.c | 3 |
9 files changed, 113 insertions, 166 deletions
diff --git a/Include/frameobject.h b/Include/frameobject.h index 55447b7..a8df445 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -27,7 +27,6 @@ typedef struct _frame { to the current stack top. */ PyObject **f_stacktop; PyObject *f_trace; /* Trace function */ - PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */ /* In a generator, we need to be able to swap between the exception state inside the generator and the exception state of the calling diff --git a/Include/genobject.h b/Include/genobject.h index 2326491..25f6c33 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -19,7 +19,7 @@ typedef struct { /* True if generator is being executed. */ char gi_running; - + /* The code object backing the generator */ PyObject *gi_code; @@ -35,6 +35,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); +PyObject *_PyGen_Send(PyGenObject *, PyObject *); #ifdef __cplusplus } diff --git a/Lib/test/test_pep380.py b/Lib/test/test_pep380.py index bdcfacd..658bcb9 100644 --- a/Lib/test/test_pep380.py +++ b/Lib/test/test_pep380.py @@ -10,7 +10,7 @@ see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Pyt import unittest import io import sys -import traceback +import inspect import parser from test.support import captured_stderr @@ -919,6 +919,27 @@ class TestPEP380Operation(unittest.TestCase): next(g1) g1.close() + def test_delegator_is_visible_to_debugger(self): + def call_stack(): + return [f[3] for f in inspect.stack()] + + def gen(): + yield call_stack() + yield call_stack() + yield call_stack() + + def spam(g): + yield from g + + def eggs(g): + yield from g + + for stack in spam(gen()): + self.assertTrue('spam' in stack) + + for stack in spam(eggs(gen())): + self.assertTrue('spam' in stack and 'eggs' in stack) + def test_main(): from test import support diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9f6af7f..2afc261 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -730,7 +730,7 @@ class SizeofTest(unittest.TestCase): nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass check(func, size(h + '12P')) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c1ec811..b33d72b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -444,7 +444,6 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(f->f_exc_type); Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); - Py_CLEAR(f->f_yieldfrom); co = f->f_code; if (co->co_zombieframe == NULL) @@ -476,7 +475,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) Py_VISIT(f->f_exc_type); Py_VISIT(f->f_exc_value); Py_VISIT(f->f_exc_traceback); - Py_VISIT(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -510,7 +508,6 @@ frame_clear(PyFrameObject *f) Py_CLEAR(f->f_exc_value); Py_CLEAR(f->f_exc_traceback); Py_CLEAR(f->f_trace); - Py_CLEAR(f->f_yieldfrom); /* locals */ slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars); @@ -714,7 +711,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, f->f_lasti = -1; f->f_lineno = code->co_firstlineno; f->f_iblock = 0; - f->f_yieldfrom = NULL; _PyObject_GC_TRACK(f); return f; diff --git a/Objects/genobject.c b/Objects/genobject.c index f25b8a5..cd2fe3d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,7 +6,6 @@ #include "opcode.h" static PyObject *gen_close(PyGenObject *gen, PyObject *args); -static void gen_undelegate(PyGenObject *gen); static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) @@ -41,15 +40,6 @@ gen_dealloc(PyGenObject *gen) PyObject_GC_Del(gen); } -static int -gen_running(PyGenObject *gen) -{ - if (gen->gi_running) { - PyErr_SetString(PyExc_ValueError, "generator already executing"); - return 1; - } - return 0; -} static PyObject * gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) @@ -58,7 +48,11 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) PyFrameObject *f = gen->gi_frame; PyObject *result; - assert(!gen->gi_running); + if (gen->gi_running) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return NULL; + } if (f==NULL || f->f_stacktop == NULL) { /* Only set exception if called from send() */ if (arg && !exc) @@ -136,45 +130,10 @@ PyDoc_STRVAR(send_doc, "send(arg) -> send 'arg' into generator,\n\ return next yielded value or raise StopIteration."); -static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +PyObject * +_PyGen_Send(PyGenObject *gen, PyObject *arg) { - int exc = 0; - PyObject *ret; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - if (gen_running(gen)) - return NULL; - /* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed? - * Or would it be valid to rely on borrowed references? - */ - Py_INCREF(arg); - if (yf) { - Py_INCREF(yf); - gen->gi_running = 1; - if (PyGen_CheckExact(yf)) { - ret = gen_send((PyGenObject *)yf, arg); - } else { - if (arg == Py_None) - ret = PyIter_Next(yf); - else - ret = PyObject_CallMethod(yf, "send", "O", arg); - } - gen->gi_running = 0; - if (ret) { - Py_DECREF(yf); - goto done; - } - gen_undelegate(gen); - Py_CLEAR(arg); - if (PyGen_FetchStopIterationValue(&arg) < 0) { - exc = 1; - } - Py_DECREF(yf); - } - ret = gen_send_ex(gen, arg, exc); -done: - Py_XDECREF(arg); - return ret; + return gen_send_ex(gen, arg, 0); } PyDoc_STRVAR(close_doc, @@ -186,49 +145,61 @@ PyDoc_STRVAR(close_doc, */ static int -gen_close_iter(PyGenObject *gen, PyObject *yf) +gen_close_iter(PyObject *yf) { PyObject *retval = NULL; - int err = 0; - + if (PyGen_CheckExact(yf)) { retval = gen_close((PyGenObject *)yf, NULL); - if (!retval) - err = -1; + if (retval == NULL) + return -1; } else { - PyObject *meth; - gen->gi_running = 1; - meth = PyObject_GetAttrString(yf, "close"); + PyObject *meth = PyObject_GetAttrString(yf, "close"); if (meth == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_WriteUnraisable(yf); - } PyErr_Clear(); } else { retval = PyObject_CallFunction(meth, ""); Py_DECREF(meth); - if (!retval) - err = -1; + if (retval == NULL) + return -1; } - gen->gi_running = 0; } Py_XDECREF(retval); - return err; -} + return 0; +} + +static PyObject * +gen_yf(PyGenObject *gen) +{ + PyObject *yf = NULL; + PyFrameObject *f = gen->gi_frame; + + if (f) { + PyObject *bytecode = f->f_code->co_code; + unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + + if (code[f->f_lasti + 1] != YIELD_FROM) + return NULL; + yf = f->f_stacktop[-1]; + Py_INCREF(yf); + } + + return yf; +} static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen_yf(gen); int err = 0; - if (gen_running(gen)) - return NULL; if (yf) { - Py_INCREF(yf); - err = gen_close_iter(gen, yf); - gen_undelegate(gen); + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; Py_DECREF(yf); } if (err == 0) @@ -241,8 +212,7 @@ gen_close(PyGenObject *gen, PyObject *args) return NULL; } if (PyErr_ExceptionMatches(PyExc_StopIteration) - || PyErr_ExceptionMatches(PyExc_GeneratorExit)) - { + || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); /* ignore these errors */ Py_INCREF(Py_None); return Py_None; @@ -323,29 +293,27 @@ gen_throw(PyGenObject *gen, PyObject *args) PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; + PyObject *yf = gen_yf(gen); if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) return NULL; - if (gen_running(gen)) - return NULL; - if (yf) { PyObject *ret; int err; - Py_INCREF(yf); if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - err = gen_close_iter(gen, yf); + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; Py_DECREF(yf); - gen_undelegate(gen); if (err < 0) return gen_send_ex(gen, Py_None, 1); goto throw_here; } - gen->gi_running = 1; if (PyGen_CheckExact(yf)) { + gen->gi_running = 1; ret = gen_throw((PyGenObject *)yf, args); + gen->gi_running = 0; } else { PyObject *meth = PyObject_GetAttrString(yf, "throw"); if (meth == NULL) { @@ -355,18 +323,22 @@ gen_throw(PyGenObject *gen, PyObject *args) } PyErr_Clear(); Py_DECREF(yf); - gen_undelegate(gen); - gen->gi_running = 0; goto throw_here; } + gen->gi_running = 1; ret = PyObject_CallObject(meth, args); + gen->gi_running = 0; Py_DECREF(meth); } - gen->gi_running = 0; Py_DECREF(yf); if (!ret) { PyObject *val; - gen_undelegate(gen); + /* Pop subiterator from stack */ + ret = *(--gen->gi_frame->f_stacktop); + assert(ret == yf); + Py_DECREF(ret); + /* Termination repetition of YIELD_FROM */ + gen->gi_frame->f_lasti++; if (PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send_ex(gen, val, 0); Py_DECREF(val); @@ -441,45 +413,12 @@ gen_iternext(PyGenObject *gen) { PyObject *val = NULL; PyObject *ret; - int exc = 0; - PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL; - if (gen_running(gen)) - return NULL; - if (yf) { - Py_INCREF(yf); - /* ceval.c ensures that yf is an iterator */ - gen->gi_running = 1; - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->gi_running = 0; - if (ret) { - Py_DECREF(yf); - return ret; - } - gen_undelegate(gen); - if (PyGen_FetchStopIterationValue(&val) < 0) - exc = 1; - Py_DECREF(yf); - } - ret = gen_send_ex(gen, val, exc); + ret = gen_send_ex(gen, val, 0); Py_XDECREF(val); return ret; } /* - * In certain recursive situations, a generator may lose its frame - * before we get a chance to clear f_yieldfrom, so we use this - * helper function. - */ - -static void -gen_undelegate(PyGenObject *gen) { - if (gen->gi_frame) { - Py_XDECREF(gen->gi_frame->f_yieldfrom); - gen->gi_frame->f_yieldfrom = NULL; - } -} - -/* * If StopIteration exception is set, fetches its 'value' * attribute if any, otherwise sets pvalue to None. * @@ -492,7 +431,7 @@ int PyGen_FetchStopIterationValue(PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; - + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { PyErr_Fetch(&et, &ev, &tb); Py_XDECREF(et); @@ -548,7 +487,7 @@ static PyMemberDef gen_memberlist[] = { }; static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, + {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc}, {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ diff --git a/Python/ceval.c b/Python/ceval.c index 017dc4a..5498056 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1170,6 +1170,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) f->f_lasti to -1 (i.e. the index *before* the first instruction) and YIELD_VALUE doesn't fiddle with f_lasti any more. So this does work. Promise. + YIELD_FROM sets f_lasti to itself, in order to repeated yield + multiple values. When the PREDICT() macros are enabled, some opcode pairs follow in direct succession without updating f->f_lasti. A successful @@ -1830,49 +1832,35 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) TARGET(YIELD_FROM) u = POP(); - x = PyObject_GetIter(u); + x = TOP(); + /* send u to x */ + if (PyGen_CheckExact(x)) { + retval = _PyGen_Send((PyGenObject *)x, u); + } else { + if (u == Py_None) + retval = PyIter_Next(x); + else + retval = PyObject_CallMethod(x, "send", "O", u); + } Py_DECREF(u); - if (x == NULL) - break; - /* x is now the iterator, make the first next() call */ - retval = (*Py_TYPE(x)->tp_iternext)(x); if (!retval) { - PyObject *et, *ev, *tb; - /* iter may be exhausted */ - Py_CLEAR(x); - if (PyErr_Occurred() && - !PyErr_ExceptionMatches(PyExc_StopIteration)) { - /* some other exception */ + PyObject *val; + x = POP(); /* Remove iter from stack */ + Py_DECREF(x); + err = PyGen_FetchStopIterationValue(&val); + if (err < 0) { + x = NULL; break; } - /* try to get return value from exception */ - PyErr_Fetch(&et, &ev, &tb); - Py_XDECREF(et); - Py_XDECREF(tb); - /* u is return value */ - u = NULL; - if (ev) { - u = PyObject_GetAttrString(ev, "value"); - Py_DECREF(ev); - if (u == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - /* some other exception */ - break; - } - PyErr_Clear(); - } - } - if (u == NULL) { - u = Py_None; - Py_INCREF(u); - } - PUSH(u); + x = val; + PUSH(x); continue; } - /* x is iterator, retval is value to be yielded */ - f->f_yieldfrom = x; + /* x remains on stack, retval is value to be yielded */ f->f_stacktop = stack_pointer; why = WHY_YIELD; + /* and repeat... */ + f->f_lasti--; goto fast_yield; TARGET(YIELD_VALUE) diff --git a/Python/compile.c b/Python/compile.c index b64c800..1722a5f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -840,9 +840,9 @@ opcode_stack_effect(int opcode, int oparg) case IMPORT_STAR: return -1; case YIELD_VALUE: - case YIELD_FROM: return 0; - + case YIELD_FROM: + return -1; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -3323,6 +3323,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e) ADDOP_O(c, LOAD_CONST, Py_None, consts); } if (e->kind == YieldFrom_kind) { + ADDOP(c, GET_ITER); + ADDOP_O(c, LOAD_CONST, Py_None, consts); ADDOP(c, YIELD_FROM); } else { diff --git a/Python/import.c b/Python/import.c index 4871b99..14148c7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,7 +104,8 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header) + 3220 (changed PEP 380 implementation) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no |