bpo-46409: Make generators in bytecode (GH-30633)

* Add RETURN_GENERATOR and JUMP_NO_INTERRUPT opcodes.

* Trim frame and generator by word each.

* Minor refactor of frame.c

* Update test.test_sys to account for smaller frames.

* Treat generator functions as normal functions when evaluating and specializing.

1 files changed, 36 insertions, 20 deletions

diff --git a/Python/compile.c b/Python/compile.c
index 86f888e..5d32959 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -969,6 +969,7 @@ stack_effect(int opcode, int oparg, int jump)
         /* Jumps */
         case JUMP_FORWARD:
         case JUMP_ABSOLUTE:
+        case JUMP_NO_INTERRUPT:
             return 0;
 
         case JUMP_IF_TRUE_OR_POP:
@@ -1017,6 +1018,9 @@ stack_effect(int opcode, int oparg, int jump)
         case DELETE_FAST:
             return 0;
 
+        case RETURN_GENERATOR:
+            return 0;
+
         case RAISE_VARARGS:
             return -oparg;
 
@@ -1841,7 +1845,7 @@ compiler_add_yield_from(struct compiler *c, int await)
     ADDOP_JUMP(c, SEND, exit);
     compiler_use_next_block(c, resume);
     ADDOP_I(c, RESUME, await ? 3 : 2);
-    ADDOP_JUMP(c, JUMP_ABSOLUTE, start);
+    ADDOP_JUMP(c, JUMP_NO_INTERRUPT, start);
     compiler_use_next_block(c, exit);
     return 1;
 }
@@ -7055,6 +7059,7 @@ stackdepth(struct compiler *c)
             }
             depth = new_depth;
             if (instr->i_opcode == JUMP_ABSOLUTE ||
+                instr->i_opcode == JUMP_NO_INTERRUPT ||
                 instr->i_opcode == JUMP_FORWARD ||
                 instr->i_opcode == RETURN_VALUE ||
                 instr->i_opcode == RAISE_VARARGS ||
@@ -7572,9 +7577,6 @@ normalize_jumps(struct assembler *a)
             if (last->i_target->b_visited == 0) {
                 last->i_opcode = JUMP_FORWARD;
             }
-            else if (b->b_iused >= 2 && b->b_instr[b->b_iused-2].i_opcode == SEND) {
-                last->i_opcode = JUMP_ABSOLUTE_QUICK;
-            }
         }
     }
 }
@@ -7998,6 +8000,34 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
     }
     assert(c->u->u_firstlineno > 0);
 
+    /* Add the generator prefix instructions. */
+    if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
+        struct instr make_gen = {
+            .i_opcode = RETURN_GENERATOR,
+            .i_oparg = 0,
+            .i_lineno = c->u->u_firstlineno,
+            .i_col_offset = -1,
+            .i_end_lineno = c->u->u_firstlineno,
+            .i_end_col_offset = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, 0, &make_gen) < 0) {
+            return -1;
+        }
+        struct instr pop_top = {
+            .i_opcode = POP_TOP,
+            .i_oparg = 0,
+            .i_lineno = -1,
+            .i_col_offset = -1,
+            .i_end_lineno = -1,
+            .i_end_col_offset = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, 1, &pop_top) < 0) {
+            return -1;
+        }
+    }
+
     /* Set up cells for any variable that escapes, to be put in a closure. */
     const int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars);
     if (ncellvars) {
@@ -8036,22 +8066,6 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
         PyMem_RawFree(sorted);
     }
 
-    /* Add the generator prefix instructions. */
-    if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
-        struct instr pop_top = {
-            .i_opcode = POP_TOP,
-            .i_oparg = 0,
-            .i_lineno = -1,
-            .i_col_offset = -1,
-            .i_end_lineno = -1,
-            .i_end_col_offset = -1,
-            .i_target = NULL,
-        };
-        if (insert_instruction(entryblock, 0, &pop_top) < 0) {
-            return -1;
-        }
-    }
-
     if (nfreevars) {
         struct instr copy_frees = {
             .i_opcode = COPY_FREE_VARS,
@@ -8801,6 +8815,7 @@ normalize_basic_block(basicblock *bb) {
                 break;
             case JUMP_ABSOLUTE:
             case JUMP_FORWARD:
+            case JUMP_NO_INTERRUPT:
                 bb->b_nofallthrough = 1;
                 /* fall through */
             case POP_JUMP_IF_NOT_NONE:
@@ -8985,6 +9000,7 @@ optimize_cfg(struct compiler *c, struct assembler *a, PyObject *consts)
         if (b->b_iused > 0) {
             struct instr *b_last_instr = &b->b_instr[b->b_iused - 1];
             if (b_last_instr->i_opcode == JUMP_ABSOLUTE ||
+                b_last_instr->i_opcode == JUMP_NO_INTERRUPT ||
                 b_last_instr->i_opcode == JUMP_FORWARD) {
                 if (b_last_instr->i_target == b->b_next) {
                     assert(b->b_next->b_iused);