From 0367a36fdc36b9c909c4d5acf7cde6ceeec0ba69 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 24 Jan 2022 11:08:53 +0000 Subject: bpo-43683: Streamline YIELD_VALUE and SEND (GH-30723) * Split YIELD_VALUE into ASYNC_GEN_WRAP; YIELD_VALUE for async generators. * Split SEND into SEND; YIELD_VALUE. * Document new opcodes. --- Doc/library/dis.rst | 16 ++++++++++ Include/opcode.h | 13 ++++---- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 2 +- .../2022-01-20-17-13-49.bpo-43683.BqQ26Z.rst | 3 ++ Objects/genobject.c | 6 ++-- Python/ceval.c | 35 +++++++++------------- Python/compile.c | 25 ++++++++++++---- Python/opcode_targets.h | 12 ++++---- 9 files changed, 72 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-01-20-17-13-49.bpo-43683.BqQ26Z.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index c9a4768..ddba668 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1233,6 +1233,22 @@ All of the following opcodes use their arguments. .. versionadded:: 3.11 +.. opcode:: SEND + + Sends ``None`` to the sub-generator of this generator. + Used in ``yield from`` and ``await`` statements. + + .. versionadded:: 3.11 + + +.. opcode:: ASYNC_GEN_WRAP + + Wraps the value on top of the stack in an ``async_generator_wrapped_value``. + Used to yield in async generators. + + .. versionadded:: 3.11 + + .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between diff --git a/Include/opcode.h b/Include/opcode.h index c0686bd..985758d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -44,6 +44,7 @@ extern "C" { #define IMPORT_STAR 84 #define SETUP_ANNOTATIONS 85 #define YIELD_VALUE 86 +#define ASYNC_GEN_WRAP 87 #define PREP_RERAISE_STAR 88 #define POP_EXCEPT 89 #define HAVE_ARGUMENT 90 @@ -165,12 +166,12 @@ extern "C" { #define STORE_ATTR_ADAPTIVE 79 #define STORE_ATTR_INSTANCE_VALUE 80 #define STORE_ATTR_SLOT 81 -#define STORE_ATTR_WITH_HINT 87 -#define LOAD_FAST__LOAD_FAST 131 -#define STORE_FAST__LOAD_FAST 140 -#define LOAD_FAST__LOAD_CONST 141 -#define LOAD_CONST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 150 +#define STORE_ATTR_WITH_HINT 131 +#define LOAD_FAST__LOAD_FAST 140 +#define STORE_FAST__LOAD_FAST 141 +#define LOAD_FAST__LOAD_CONST 143 +#define LOAD_CONST__LOAD_FAST 150 +#define STORE_FAST__STORE_FAST 153 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 1560e60..cd4f69c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -381,6 +381,7 @@ _code_type = type(_write_atomic.__code__) # Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) # Python 3.11a4 3474 (Add RESUME opcode) # Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) +# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) # Python 3.12 will start with magic number 3500 @@ -394,7 +395,7 @@ _code_type = type(_write_atomic.__code__) # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3475).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3476).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 73b41d2..1bd48ee 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -101,7 +101,7 @@ def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) def_op('SETUP_ANNOTATIONS', 85) def_op('YIELD_VALUE', 86) - +def_op('ASYNC_GEN_WRAP', 87) def_op('PREP_RERAISE_STAR', 88) def_op('POP_EXCEPT', 89) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-20-17-13-49.bpo-43683.BqQ26Z.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-20-17-13-49.bpo-43683.BqQ26Z.rst new file mode 100644 index 0000000..737f44f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-20-17-13-49.bpo-43683.BqQ26Z.rst @@ -0,0 +1,3 @@ +Add ASYNC_GEN_WRAP opcode to wrap the value to be yielded in async +generators. Removes the need to special case async generators in the +``YIELD_VALUE`` instruction. diff --git a/Objects/genobject.c b/Objects/genobject.c index 46b0190..b2d402e 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -353,7 +353,7 @@ _PyGen_yf(PyGenObject *gen) PyObject *bytecode = gen->gi_code->co_code; unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); - if (frame->f_lasti < 0) { + if (frame->f_lasti < 1) { /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ @@ -361,7 +361,7 @@ _PyGen_yf(PyGenObject *gen) return NULL; } - if (code[frame->f_lasti*sizeof(_Py_CODEUNIT)] != SEND || frame->stacktop < 0) + if (code[(frame->f_lasti-1)*sizeof(_Py_CODEUNIT)] != SEND || frame->stacktop < 0) return NULL; yf = _PyFrame_StackPeek(frame); Py_INCREF(yf); @@ -488,6 +488,8 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, assert(frame->f_lasti >= 0); PyObject *bytecode = gen->gi_code->co_code; unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + /* Backup to SEND */ + frame->f_lasti--; assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND); int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1]; frame->f_lasti += jump; diff --git a/Python/ceval.c b/Python/ceval.c index 9aaddd9..2c524ab 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2650,32 +2650,25 @@ handle_eval_breaker: } assert (gen_status == PYGEN_NEXT); assert (retval != NULL); - frame->f_state = FRAME_SUSPENDED; - _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); - _Py_LeaveRecursiveCall(tstate); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; - assert(tstate->cframe->current_frame == frame->previous); - assert(!_PyErr_Occurred(tstate)); - return retval; + PUSH(retval); + DISPATCH(); + } + + TARGET(ASYNC_GEN_WRAP) { + PyObject *v = TOP(); + assert(frame->f_code->co_flags & CO_ASYNC_GENERATOR); + PyObject *w = _PyAsyncGenValueWrapperNew(v); + if (w == NULL) { + goto error; + } + SET_TOP(w); + Py_DECREF(v); + DISPATCH(); } TARGET(YIELD_VALUE) { assert(frame->is_entry); PyObject *retval = POP(); - - if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) { - PyObject *w = _PyAsyncGenValueWrapperNew(retval); - Py_DECREF(retval); - if (w == NULL) { - retval = NULL; - goto error; - } - retval = w; - } frame->f_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); diff --git a/Python/compile.c b/Python/compile.c index 5d32959..feb9fca 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -910,6 +910,7 @@ stack_effect(int opcode, int oparg, int jump) return -1; case SETUP_ANNOTATIONS: return 0; + case ASYNC_GEN_WRAP: case YIELD_VALUE: return 0; case POP_BLOCK: @@ -1541,6 +1542,9 @@ compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b) #define POP_EXCEPT_AND_RERAISE(C) \ RETURN_IF_FALSE(compiler_pop_except_and_reraise((C))) +#define ADDOP_YIELD(C) \ + RETURN_IF_FALSE(addop_yield(C)) + #define VISIT(C, TYPE, V) {\ if (!compiler_visit_ ## TYPE((C), (V))) \ return 0; \ @@ -1844,6 +1848,7 @@ compiler_add_yield_from(struct compiler *c, int await) compiler_use_next_block(c, start); ADDOP_JUMP(c, SEND, exit); compiler_use_next_block(c, resume); + ADDOP(c, YIELD_VALUE); ADDOP_I(c, RESUME, await ? 3 : 2); ADDOP_JUMP(c, JUMP_NO_INTERRUPT, start); compiler_use_next_block(c, exit); @@ -4094,6 +4099,17 @@ addop_binary(struct compiler *c, operator_ty binop, bool inplace) return 1; } + +static int +addop_yield(struct compiler *c) { + if (c->u->u_ste->ste_generator && c->u->u_ste->ste_coroutine) { + ADDOP(c, ASYNC_GEN_WRAP); + } + ADDOP(c, YIELD_VALUE); + ADDOP_I(c, RESUME, 1); + return 1; +} + static int compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx) { @@ -5144,8 +5160,7 @@ compiler_sync_comprehension_generator(struct compiler *c, switch (type) { case COMP_GENEXP: VISIT(c, expr, elt); - ADDOP(c, YIELD_VALUE); - ADDOP_I(c, RESUME, 1); + ADDOP_YIELD(c); ADDOP(c, POP_TOP); break; case COMP_LISTCOMP: @@ -5243,8 +5258,7 @@ compiler_async_comprehension_generator(struct compiler *c, switch (type) { case COMP_GENEXP: VISIT(c, expr, elt); - ADDOP(c, YIELD_VALUE); - ADDOP_I(c, RESUME, 1); + ADDOP_YIELD(c); ADDOP(c, POP_TOP); break; case COMP_LISTCOMP: @@ -5714,8 +5728,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) else { ADDOP_LOAD_CONST(c, Py_None); } - ADDOP(c, YIELD_VALUE); - ADDOP_I(c, RESUME, 1); + ADDOP_YIELD(c); break; case YieldFrom_kind: if (c->u->u_ste->ste_type != FunctionBlock) diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 11ac0e9..c19cd0e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -86,7 +86,7 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -130,7 +130,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_NO_INTERRUPT, @@ -139,20 +139,20 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, -- cgit v0.12