bpo-46409: Make generators in bytecode (GH-30633)

* Add RETURN_GENERATOR and JUMP_NO_INTERRUPT opcodes.

* Trim frame and generator by word each.

* Minor refactor of frame.c

* Update test.test_sys to account for smaller frames.

* Treat generator functions as normal functions when evaluating and specializing.

2 files changed, 43 insertions, 24 deletions

diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 4dd2183..81ad4cc 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -242,6 +242,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
                     break;
                 }
                 case JUMP_ABSOLUTE:
+                case JUMP_NO_INTERRUPT:
                     j = get_arg(code, i);
                     assert(j < len);
                     if (stacks[j] == UNINITIALIZED && j < i) {
@@ -625,7 +626,7 @@ frame_dealloc(PyFrameObject *f)
 {
     /* It is the responsibility of the owning generator/coroutine
      * to have cleared the generator pointer */
-    assert(f->f_frame->generator == NULL);
+    assert(!f->f_frame->is_generator);
 
     if (_PyObject_GC_IS_TRACKED(f)) {
         _PyObject_GC_UNTRACK(f);
@@ -698,8 +699,11 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
                         "cannot clear an executing frame");
         return NULL;
     }
-    if (f->f_frame->generator) {
-        _PyGen_Finalize(f->f_frame->generator);
+    if (f->f_frame->is_generator) {
+        assert(!f->f_owns_frame);
+        size_t offset_in_gen = offsetof(PyGenObject, gi_iframe);
+        PyObject *gen = (PyObject *)(((char *)f->f_frame) - offset_in_gen);
+        _PyGen_Finalize(gen);
     }
     (void)frame_tp_clear(f);
     Py_RETURN_NONE;
diff --git a/Objects/genobject.c b/Objects/genobject.c
index d093f3d..46b0190 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -87,7 +87,7 @@ _PyGen_Finalize(PyObject *self)
        issue a RuntimeWarning. */
     if (gen->gi_code != NULL &&
         ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE &&
-        ((InterpreterFrame *)gen->gi_iframe)->f_lasti == -1)
+        ((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_CREATED)
     {
         _PyErr_WarnUnawaitedCoroutine((PyObject *)gen);
     }
@@ -133,7 +133,7 @@ gen_dealloc(PyGenObject *gen)
     if (gen->gi_frame_valid) {
         InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
         gen->gi_frame_valid = 0;
-        frame->generator = NULL;
+        frame->is_generator = false;
         frame->previous = NULL;
         _PyFrame_Clear(frame);
     }
@@ -156,7 +156,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     PyObject *result;
 
     *presult = NULL;
-    if (frame->f_lasti < 0 && arg && arg != Py_None) {
+    if (frame->f_state == FRAME_CREATED && arg && arg != Py_None) {
         const char *msg = "can't send non-None value to a "
                             "just-started generator";
         if (PyCoro_CheckExact(gen)) {
@@ -265,7 +265,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     /* first clean reference cycle through stored exception traceback */
     _PyErr_ClearExcState(&gen->gi_exc_state);
 
-    frame->generator = NULL;
+    frame->is_generator = false;
     gen->gi_frame_valid = 0;
     _PyFrame_Clear(frame);
     *presult = result;
@@ -754,6 +754,15 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored))
 }
 
 static PyObject *
+gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored))
+{
+    if (gen->gi_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_SUSPENDED);
+}
+
+static PyObject *
 _gen_getframe(PyGenObject *gen, const char *const name)
 {
     if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) {
@@ -780,6 +789,7 @@ static PyGetSetDef gen_getsetlist[] = {
      PyDoc_STR("object being iterated by yield from, or None")},
     {"gi_running", (getter)gen_getrunning, NULL, NULL},
     {"gi_frame", (getter)gen_getframe,  NULL, NULL},
+    {"gi_suspended", (getter)gen_getsuspended,  NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -886,22 +896,16 @@ make_gen(PyTypeObject *type, PyFunctionObject *func)
     gen->gi_weakreflist = NULL;
     gen->gi_exc_state.exc_value = NULL;
     gen->gi_exc_state.previous_item = NULL;
-    if (func->func_name != NULL)
-        gen->gi_name = func->func_name;
-    else
-        gen->gi_name = gen->gi_code->co_name;
-    Py_INCREF(gen->gi_name);
-    if (func->func_qualname != NULL)
-        gen->gi_qualname = func->func_qualname;
-    else
-        gen->gi_qualname = gen->gi_name;
-    Py_INCREF(gen->gi_qualname);
+    assert(func->func_name != NULL);
+    gen->gi_name = Py_NewRef(func->func_name);
+    assert(func->func_qualname != NULL);
+    gen->gi_qualname = Py_NewRef(func->func_qualname);
     _PyObject_GC_TRACK(gen);
     return (PyObject *)gen;
 }
 
 static PyObject *
-compute_cr_origin(int origin_depth);
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame);
 
 PyObject *
 _Py_MakeCoro(PyFunctionObject *func)
@@ -935,7 +939,8 @@ _Py_MakeCoro(PyFunctionObject *func)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        assert(_PyEval_GetFrame());
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()->previous);
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
@@ -965,7 +970,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
     assert(frame->frame_obj == f);
     f->f_owns_frame = 0;
     f->f_frame = frame;
-    frame->generator = (PyObject *) gen;
+    frame->is_generator = true;
     assert(PyObject_GC_IsTracked((PyObject *)f));
     gen->gi_code = PyFrame_GetCode(f);
     Py_INCREF(gen->gi_code);
@@ -1098,6 +1103,15 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored))
 }
 
 static PyObject *
+cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored))
+{
+    if (coro->cr_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)coro->cr_iframe)->f_state == FRAME_SUSPENDED);
+}
+
+static PyObject *
 cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored))
 {
     if (coro->cr_frame_valid == 0) {
@@ -1122,6 +1136,7 @@ static PyGetSetDef coro_getsetlist[] = {
      PyDoc_STR("object being awaited on, or None")},
     {"cr_running", (getter)cr_getrunning, NULL, NULL},
     {"cr_frame", (getter)cr_getframe, NULL, NULL},
+    {"cr_suspended", (getter)cr_getsuspended, NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -1299,9 +1314,9 @@ PyTypeObject _PyCoroWrapper_Type = {
 };
 
 static PyObject *
-compute_cr_origin(int origin_depth)
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame)
 {
-    InterpreterFrame *frame = _PyEval_GetFrame();
+    InterpreterFrame *frame = current_frame;
     /* First count how many frames we have */
     int frame_count = 0;
     for (; frame && frame_count < origin_depth; ++frame_count) {
@@ -1313,7 +1328,7 @@ compute_cr_origin(int origin_depth)
     if (cr_origin == NULL) {
         return NULL;
     }
-    frame = _PyEval_GetFrame();
+    frame = current_frame;
     for (int i = 0; i < frame_count; ++i) {
         PyCodeObject *code = frame->f_code;
         PyObject *frameinfo = Py_BuildValue("OiO",
@@ -1345,7 +1360,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame());
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);