From 67a91f78e4395148afcc33e5cd6f3f0a9623e63a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:43:10 +0100 Subject: gh-109094: replace frame->prev_instr by frame->instr_ptr (#109095) --- Include/internal/pycore_frame.h | 22 ++--- Include/internal/pycore_opcode_metadata.h | 16 +-- Include/internal/pycore_runtime.h | 2 +- Include/internal/pycore_runtime_init.h | 2 +- .../2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst | 3 + Objects/frame_layout.md | 25 +++++ Objects/frameobject.c | 8 +- Objects/genobject.c | 13 ++- Python/abstract_interp_cases.c.h | 4 + Python/bytecodes.c | 99 ++++++++++++------- Python/ceval.c | 30 +++--- Python/ceval_macros.h | 16 +-- Python/executor.c | 5 +- Python/executor_cases.c.h | 20 ++-- Python/frame.c | 2 +- Python/generated_cases.c.h | 107 +++++++++++++-------- Python/instrumentation.c | 8 +- Python/optimizer.c | 10 +- Python/specialize.c | 3 +- Tools/cases_generator/analysis.py | 4 +- Tools/cases_generator/generate_cases.py | 8 +- Tools/cases_generator/stacking.py | 2 +- Tools/gdb/libpython.py | 4 +- 23 files changed, 249 insertions(+), 164 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index bd44827..ec97030 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -58,26 +58,16 @@ typedef struct _PyInterpreterFrame { PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ - // NOTE: This is not necessarily the last instruction started in the given - // frame. Rather, it is the code unit *prior to* the *next* instruction. For - // example, it may be an inline CACHE entry, an instruction we just jumped - // over, or (in the case of a newly-created frame) a totally invalid value: - _Py_CODEUNIT *prev_instr; + _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ int stacktop; /* Offset of TOS from localsplus */ - /* The return_offset determines where a `RETURN` should go in the caller, - * relative to `prev_instr`. - * It is only meaningful to the callee, - * so it needs to be set in any CALL (to a Python function) - * or SEND (to a coroutine or generator). - * If there is no callee, then it is meaningless. */ - uint16_t return_offset; + uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ PyObject *localsplus[1]; } _PyInterpreterFrame; #define _PyInterpreterFrame_LASTI(IF) \ - ((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF)))) + ((int)((IF)->instr_ptr - _PyCode_CODE(_PyFrame_GetCode(IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { assert(PyCode_Check(f->f_executable)); @@ -134,7 +124,7 @@ _PyFrame_Initialize( frame->f_locals = locals; frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) - 1; + frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; @@ -185,7 +175,7 @@ _PyFrame_IsIncomplete(_PyInterpreterFrame *frame) return true; } return frame->owner != FRAME_OWNED_BY_GENERATOR && - frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; + frame->instr_ptr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable; } static inline _PyInterpreterFrame * @@ -297,7 +287,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code); + frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; frame->return_offset = 0; return frame; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 938597d..e2ed9ba 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -87,7 +87,7 @@ #define _POP_JUMP_IF_FALSE 359 #define _POP_JUMP_IF_TRUE 360 #define _JUMP_TO_TOP 361 -#define _SAVE_CURRENT_IP 362 +#define _SAVE_RETURN_OFFSET 362 #define _INSERT 363 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -654,7 +654,7 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case _SET_IP: return 0; - case _SAVE_CURRENT_IP: + case _SAVE_RETURN_OFFSET: return 0; case _EXIT_TRACE: return 0; @@ -1230,7 +1230,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case _SET_IP: return 0; - case _SAVE_CURRENT_IP: + case _SAVE_RETURN_OFFSET: return 0; case _EXIT_TRACE: return 0; @@ -1296,7 +1296,7 @@ struct opcode_macro_expansion { #define OPARG_CACHE_4 4 #define OPARG_TOP 5 #define OPARG_BOTTOM 6 -#define OPARG_SET_IP 7 +#define OPARG_SAVE_RETURN_OFFSET 7 #define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) #define SAME_OPCODE_METADATA(OP1, OP2) \ @@ -1589,7 +1589,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SAVE_CURRENT_IP] = { true, INSTR_FMT_IX, 0 }, + [_SAVE_RETURN_OFFSET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, }; @@ -1719,8 +1719,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, @@ -1812,7 +1812,7 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", - [_SAVE_CURRENT_IP] = "_SAVE_CURRENT_IP", + [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_INSERT] = "_INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f2383b4..ec026d4 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -87,7 +87,7 @@ typedef struct _Py_DebugOffsets { struct _interpreter_frame { off_t previous; off_t executable; - off_t prev_instr; + off_t instr_ptr; off_t localsplus; off_t owner; } interpreter_frame; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 4a48907..73fa5f5 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -58,7 +58,7 @@ extern PyTypeObject _PyExc_MemoryError; .interpreter_frame = { \ .previous = offsetof(_PyInterpreterFrame, previous), \ .executable = offsetof(_PyInterpreterFrame, f_executable), \ - .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \ + .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst new file mode 100644 index 0000000..332afb6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-13-16-55-55.gh-issue-109094.ziL4cJ.rst @@ -0,0 +1,3 @@ +Replace ``prev_instr`` on the interpreter frame by ``instr_ptr`` which +points to the beginning of the instruction that is currently executing (or +will execute once the frame resumes). diff --git a/Objects/frame_layout.md b/Objects/frame_layout.md index 2f95214..b348e85 100644 --- a/Objects/frame_layout.md +++ b/Objects/frame_layout.md @@ -130,3 +130,28 @@ returns. This extra frame is inserted so that `RETURN_VALUE`, `YIELD_VALUE`, and `RETURN_GENERATOR` do not need to check whether the current frame is the entry frame. The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. + + +### The Instruction Pointer + +`_PyInterpreterFrame` has two fields which are used to maintain the instruction +pointer: `instr_ptr` and `return_offset`. + +When a frame is executing, `instr_ptr` points to the instruction currently being +executed. In a suspended frame, it points to the instruction that would execute +if the frame were to resume. After `frame.f_lineno` is set, `instr_ptr` points to +the next instruction to be executed. During a call to a python function, +`instr_ptr` points to the call instruction, because this is what we would expect +to see in an exception traceback. + +The `return_offset` field determines where a `RETURN` should go in the caller, +relative to `instr_ptr`. It is only meaningful to the callee, so it needs to +be set in any instruction that implements a call (to a Python function), +including CALL, SEND and BINARY_SUBSCR_GETITEM, among others. If there is no +callee, then return_offset is meaningless. It is necessary to have a separate +field for the return offset because (1) if we apply this offset to `instr_ptr` +while executing the `RETURN`, this is too early and would lose us information +about the previous instruction which we could need for introspecting and +debugging. (2) `SEND` needs to pass two offsets to the generator: one for +`RETURN` and one for `YIELD`. It uses the `oparg` for one, and the +`return_offset` for the other. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 12faef0..698e884 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -821,7 +821,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } /* Finally set the new lasti and return OK. */ f->f_lineno = 0; - f->f_frame->prev_instr = _PyCode_CODE(code) + best_addr; + f->f_frame->instr_ptr = _PyCode_CODE(code) + best_addr; return 0; } @@ -1079,7 +1079,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; // This frame needs to be "complete", so pretend that the first RESUME ran: - f->f_frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + f->f_frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; assert(!_PyFrame_IsIncomplete(f->f_frame)); Py_DECREF(func); _PyObject_GC_TRACK(f); @@ -1093,7 +1093,7 @@ _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg) assert(_PyOpcode_Deopt[opcode] == opcode); int check_oparg = 0; for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame)); - instruction < frame->prev_instr; instruction++) + instruction < frame->instr_ptr; instruction++) { int check_opcode = _PyOpcode_Deopt[instruction->op.code]; check_oparg |= instruction->op.arg; @@ -1135,7 +1135,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) frame->localsplus[offset + i] = Py_NewRef(o); } // COPY_FREE_VARS doesn't have inline CACHEs, either: - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 6794cc6..092fd5f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -9,6 +9,7 @@ #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -362,8 +363,7 @@ _PyGen_yf(PyGenObject *gen) assert(_PyCode_CODE(_PyGen_GetCode(gen))[0].op.code != SEND); return NULL; } - _Py_CODEUNIT next = frame->prev_instr[1]; - if (!is_resume(&next) || next.op.arg < 2) + if (!is_resume(frame->instr_ptr) || frame->instr_ptr->op.arg < 2) { /* Not in a yield from */ return NULL; @@ -398,9 +398,12 @@ gen_close(PyGenObject *gen, PyObject *args) _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; /* It is possible for the previous instruction to not be a * YIELD_VALUE if the debugger has changed the lineno. */ - if (err == 0 && is_yield(frame->prev_instr)) { - assert(is_resume(frame->prev_instr + 1)); - int exception_handler_depth = frame->prev_instr[0].op.arg; + assert(_PyOpcode_Caches[YIELD_VALUE] == 0); + assert(_PyOpcode_Caches[INSTRUMENTED_YIELD_VALUE] == 0); + if (err == 0 && is_yield(frame->instr_ptr - 1)) { + _Py_CODEUNIT *yield_instr = frame->instr_ptr - 1; + assert(is_resume(frame->instr_ptr)); + int exception_handler_depth = yield_instr->op.arg; assert(exception_handler_depth > 0); /* We can safely ignore the outermost try block * as it automatically generated to handle diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 44115da..232c569 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -920,6 +920,10 @@ break; } + case _SAVE_RETURN_OFFSET: { + break; + } + case _EXIT_TRACE: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2ff3805..e101efa 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -183,9 +183,9 @@ dummy_func( tstate, oparg > 0, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->prev_instr != next_instr-1) { + if (frame->instr_ptr != next_instr-1) { /* Instrumentation has jumped */ - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -657,7 +657,8 @@ dummy_func( new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -790,10 +791,9 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -819,8 +819,8 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -843,8 +843,8 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -980,7 +980,8 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1018,13 +1019,15 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1039,6 +1042,9 @@ dummy_func( frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1048,6 +1054,7 @@ dummy_func( // or throw() call. assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ assert(frame != &entry_frame); + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1058,6 +1065,9 @@ dummy_func( frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1071,7 +1081,7 @@ dummy_func( if (oparg) { PyObject *lasti = values[0]; if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2009,7 +2019,8 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2036,7 +2047,8 @@ dummy_func( new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2304,13 +2316,14 @@ dummy_func( _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; + frame->instr_ptr = next_instr; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } + next_instr = frame->instr_ptr; goto resume_frame; } @@ -2476,7 +2489,7 @@ dummy_func( } inst(INSTRUMENTED_FOR_ITER, ( -- )) { - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2672,7 +2685,8 @@ dummy_func( SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -2983,7 +2997,8 @@ dummy_func( goto error; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3070,7 +3085,6 @@ dummy_func( op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -3078,7 +3092,7 @@ dummy_func( frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -3095,7 +3109,7 @@ dummy_func( _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; macro(CALL_PY_EXACT_ARGS) = @@ -3104,7 +3118,7 @@ dummy_func( _CHECK_FUNCTION_EXACT_ARGS + _CHECK_STACK_SPACE + _INIT_CALL_PY_EXACT_ARGS + - _SAVE_CURRENT_IP + // Sets frame->prev_instr + _SAVE_RETURN_OFFSET + _PUSH_FRAME; inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { @@ -3138,7 +3152,8 @@ dummy_func( // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3203,7 +3218,7 @@ dummy_func( Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; @@ -3215,8 +3230,8 @@ dummy_func( init_frame->localsplus[i+1] = args[i]; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); /* Link frames */ @@ -3585,7 +3600,8 @@ dummy_func( if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3681,7 +3697,8 @@ dummy_func( if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); @@ -3744,6 +3761,7 @@ dummy_func( assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; @@ -3754,6 +3772,7 @@ dummy_func( _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -3836,18 +3855,20 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT *here = frame->instr_ptr; + INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { + _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3859,7 +3880,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -3870,7 +3891,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -3888,7 +3909,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { @@ -3943,16 +3964,20 @@ dummy_func( op(_SET_IP, (--)) { TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + frame->instr_ptr = ip_offset + oparg; } - op(_SAVE_CURRENT_IP, (--)) { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + op(_SAVE_RETURN_OFFSET, (--)) { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } op(_EXIT_TRACE, (--)) { - frame->prev_instr--; // Back up to just before destination + TIER_TWO_ONLY _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/ceval.c b/Python/ceval.c index ac40425..61f8387 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -640,7 +640,9 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ { .op.code = RESUME, .op.arg = 0 } }; @@ -698,7 +700,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.f_builtins = (PyObject*)0xaaa4; #endif entry_frame.f_executable = Py_None; - entry_frame.prev_instr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stacktop = 0; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; @@ -722,7 +724,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, * we need to update instrumentation */ _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - monitor_throw(tstate, frame, frame->prev_instr); + monitor_throw(tstate, frame, frame->instr_ptr); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -733,19 +735,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; -/* Sets the above local variables from the frame */ -#define SET_LOCALS_FROM_FRAME() \ - /* Jump back to the last instruction executed... */ \ - next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } + next_instr = frame->instr_ptr; resume_frame: - SET_LOCALS_FROM_FRAME(); + stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); @@ -774,7 +772,7 @@ resume_frame: #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, - * because it needs to capture frame->prev_instr before it is updated, + * because it needs to capture frame->instr_ptr before it is updated, * as happens in the standard instruction prologue. */ #if USE_COMPUTED_GOTOS @@ -783,8 +781,8 @@ resume_frame: case INSTRUMENTED_LINE: #endif { - _Py_CODEUNIT *prev = frame->prev_instr; - _Py_CODEUNIT *here = frame->prev_instr = next_instr; + _Py_CODEUNIT *prev = frame->instr_ptr; + _Py_CODEUNIT *here = frame->instr_ptr = next_instr; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here, prev); @@ -793,7 +791,7 @@ resume_frame: next_instr = here+1; goto error; } - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; if (next_instr != here) { DISPATCH(); } @@ -908,7 +906,8 @@ exception_unwind: Python main loop. */ PyObject *exc = _PyErr_GetRaisedException(tstate); PUSH(exc); - JUMPTO(handler); + next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { goto exception_unwind; } @@ -939,7 +938,8 @@ exit_unwind: } resume_with_error: - SET_LOCALS_FROM_FRAME(); + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); goto error; } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bd28126..544e8ef 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -62,13 +62,16 @@ #ifdef Py_STATS #define INSTRUCTION_START(op) \ do { \ - frame->prev_instr = next_instr++; \ + frame->instr_ptr = next_instr++; \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else -#define INSTRUCTION_START(op) (frame->prev_instr = next_instr++) +#define INSTRUCTION_START(op) \ + do { \ + frame->instr_ptr = next_instr++; \ + } while(0) #endif #if USE_COMPUTED_GOTOS @@ -107,7 +110,6 @@ do { \ assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ @@ -146,7 +148,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) -#define JUMPTO(x) (next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + (x)) /* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is * for advancing to the next instruction, taking into account cache entries @@ -381,8 +382,9 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { #if TIER_ONE -#define LOAD_IP() \ -do { next_instr = frame->prev_instr+1; } while (0) +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) #define STORE_SP() \ _PyFrame_SetStackPointer(frame, stack_pointer) @@ -395,7 +397,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); #if TIER_TWO -#define LOAD_IP() \ +#define LOAD_IP(UNUSED) \ do { ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; } while (0) #define STORE_SP() \ diff --git a/Python/executor.c b/Python/executor.c index 1630f01..bfa7f7e 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -62,7 +62,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname), PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename), _PyFrame_GetCode(frame)->co_firstlineno, - 2 * (long)(frame->prev_instr + 1 - + 2 * (long)(frame->instr_ptr - (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive)); PyThreadState *tstate = _PyThreadState_GET(); @@ -131,6 +131,7 @@ error: // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return NULL; @@ -140,7 +141,7 @@ deoptimize: // This presumes nothing was popped from the stack (nor pushed). DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - frame->prev_instr--; // Back up to just before destination + frame->return_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return frame; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 119e77b..9f37cd7 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -692,10 +692,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -2587,7 +2586,6 @@ STACK_SHRINK(1); // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -2595,7 +2593,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -3271,12 +3269,22 @@ case _SET_IP: { TIER_TWO_ONLY - frame->prev_instr = ip_offset + oparg; + frame->instr_ptr = ip_offset + oparg; + break; + } + + case _SAVE_RETURN_OFFSET: { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif break; } case _EXIT_TRACE: { - frame->prev_instr--; // Back up to just before destination + TIER_TWO_ONLY _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/frame.c b/Python/frame.c index b483903..2865b2e 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -90,7 +90,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) // This may be a newly-created generator or coroutine frame. Since it's // dead anyways, just pretend that the first RESUME ran: PyCodeObject *code = _PyFrame_GetCode(frame); - frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable; + frame->instr_ptr = _PyCode_CODE(code) + code->_co_firsttraceable + 1; } assert(!_PyFrame_IsIncomplete(frame)); assert(f->f_back == NULL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 506c02a..3ae1a58 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -61,9 +61,9 @@ tstate, oparg > 0, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - if (frame->prev_instr != next_instr-1) { + if (frame->instr_ptr != next_instr-1) { /* Instrumentation has jumped */ - next_instr = frame->prev_instr; + next_instr = frame->instr_ptr; DISPATCH(); } } @@ -804,7 +804,8 @@ new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -999,10 +1000,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -1028,8 +1028,8 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -1054,10 +1054,9 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); LOAD_SP(); - LOAD_IP(); + LOAD_IP(frame->return_offset); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -1083,8 +1082,8 @@ _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -1239,7 +1238,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1284,7 +1284,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1293,6 +1294,7 @@ retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1307,6 +1309,9 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1318,6 +1323,7 @@ // or throw() call. assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ assert(frame != &entry_frame); + frame->instr_ptr = next_instr; PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); @@ -1328,6 +1334,9 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } @@ -1349,7 +1358,7 @@ if (oparg) { PyObject *lasti = values[0]; if (PyLong_Check(lasti)) { - frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); + frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); } else { @@ -2578,7 +2587,8 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2610,7 +2620,8 @@ new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2986,13 +2997,14 @@ _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); JUMPBY(1-original_oparg); - frame->prev_instr = next_instr - 1; + frame->instr_ptr = next_instr; Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { frame = tstate->current_frame; goto resume_with_error; } + next_instr = frame->instr_ptr; goto resume_frame; } @@ -3259,7 +3271,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3427,7 +3439,8 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - frame->return_offset = oparg; + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg; DISPATCH_INLINED(gen_frame); } @@ -3815,7 +3828,8 @@ goto error; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -3915,11 +3929,15 @@ new_frame->localsplus[i] = args[i]; } } - // _SAVE_CURRENT_IP + // _SAVE_RETURN_OFFSET next_instr += 3; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); @@ -3927,7 +3945,6 @@ { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -3935,7 +3952,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -3988,11 +4005,15 @@ new_frame->localsplus[i] = args[i]; } } - // _SAVE_CURRENT_IP + // _SAVE_RETURN_OFFSET next_instr += 3; { - TIER_ONE_ONLY - frame->prev_instr = next_instr - 1; + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); @@ -4000,7 +4021,6 @@ { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. - frame->return_offset = 0; assert(tstate->interp->eval_frame == NULL); STORE_SP(); new_frame->previous = frame; @@ -4008,7 +4028,7 @@ frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(); + LOAD_IP(0); #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { @@ -4057,7 +4077,8 @@ // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -4164,7 +4185,7 @@ Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); - assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[1].op.code == EXIT_INIT_CHECK); + assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; @@ -4176,8 +4197,8 @@ init_frame->localsplus[i+1] = args[i]; } SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - frame->return_offset = 0; + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); /* Link frames */ @@ -4685,7 +4706,8 @@ if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ @@ -4793,7 +4815,8 @@ if (new_frame == NULL) { goto error; } - frame->return_offset = 0; + assert(next_instr - frame->instr_ptr == 1); + frame->return_offset = 1; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); @@ -4874,6 +4897,7 @@ assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->instr_ptr = next_instr; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; @@ -4884,6 +4908,7 @@ _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); + LOAD_IP(frame->return_offset); goto resume_frame; } @@ -5021,20 +5046,22 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT *here = frame->instr_ptr; + INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { + _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -5048,7 +5075,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION @@ -5061,7 +5088,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr - 1; + _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -5081,7 +5108,7 @@ TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { PyObject *value = POP(); - _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4bb57a6..5fd65d5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1073,7 +1073,7 @@ _Py_call_instrumentation_jump( { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); - assert(frame->prev_instr == instr); + assert(frame->instr_ptr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); @@ -1086,9 +1086,9 @@ _Py_call_instrumentation_jump( if (err) { return NULL; } - if (frame->prev_instr != instr) { + if (frame->instr_ptr != instr) { /* The callback has caused a jump (by setting the line number) */ - return frame->prev_instr; + return frame->instr_ptr; } return target; } @@ -1138,7 +1138,6 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { - assert(frame->prev_instr == instr); PyCodeObject *code = _PyFrame_GetCode(frame); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); @@ -1153,6 +1152,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); assert(line >= 0); + assert(prev != NULL); int prev_index = (int)(prev - _PyCode_CODE(code)); int prev_line = _Py_Instrumentation_GetLine(code, prev_index); if (prev_line == line) { diff --git a/Python/optimizer.c b/Python/optimizer.c index b9aab46..6402287 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -251,7 +251,7 @@ counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject ** { ((_PyCounterExecutorObject *)self)->optimizer->count++; _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = ((_PyCounterExecutorObject *)self)->next_instr - 1; + frame->instr_ptr = ((_PyCounterExecutorObject *)self)->next_instr; Py_DECREF(self); return frame; } @@ -701,11 +701,9 @@ pop_jump_if_bool: case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; - case OPARG_SET_IP: // uop=_SET_IP; oparg=next_instr-1 - // The number of caches is smuggled in via offset: - assert(offset == _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); - oparg = INSTR_IP(instr + offset, code); - uop = _SET_IP; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); break; default: diff --git a/Python/specialize.c b/Python/specialize.c index 8e24760..cd26ff5 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2524,7 +2524,7 @@ static const PyBytesObject no_location = { }; const struct _PyCode_DEF(8) _Py_InitCleanup = { - _PyVarObject_HEAD_INIT(&PyCode_Type, 4) + _PyVarObject_HEAD_INIT(&PyCode_Type, 3) .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), @@ -2539,7 +2539,6 @@ const struct _PyCode_DEF(8) _Py_InitCleanup = { .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, .co_code_adaptive = { - NOP, 0, EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, RESUME, 0, diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index b2fa020..53f715d 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -372,8 +372,8 @@ class Analyzer: case Instruction() as instr: part, offset = self.analyze_instruction(instr, offset) parts.append(part) - if instr.name != "_SET_IP": - # _SET_IP in a macro is a no-op in Tier 1 + if instr.name != "_SAVE_RETURN_OFFSET": + # _SAVE_RETURN_OFFSET's oparg does not transfer flags.add(instr.instr_flags) case _: assert_never(component) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 04d617b..ef96ed3 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -67,7 +67,7 @@ OPARG_SIZES = { "OPARG_CACHE_4": 4, "OPARG_TOP": 5, "OPARG_BOTTOM": 6, - "OPARG_SET_IP": 7, + "OPARG_SAVE_RETURN_OFFSET": 7, } INSTR_FMT_PREFIX = "INSTR_FMT_" @@ -658,7 +658,7 @@ class Generator(Analyzer): for part in parts: if isinstance(part, Component): # All component instructions must be viable uops - if not part.instr.is_viable_uop() and part.instr.name != "_SAVE_CURRENT_IP": + if not part.instr.is_viable_uop(): # This note just reminds us about macros that cannot # be expanded to Tier 2 uops. It is not an error. # It is sometimes emitted for macros that have a @@ -671,8 +671,8 @@ class Generator(Analyzer): ) return if not part.active_caches: - if part.instr.name == "_SAVE_CURRENT_IP": - size, offset = OPARG_SIZES["OPARG_SET_IP"], cache_offset - 1 + if part.instr.name == "_SAVE_RETURN_OFFSET": + size, offset = OPARG_SIZES["OPARG_SAVE_RETURN_OFFSET"], cache_offset else: size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index bba2db8..69a6c10 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -448,7 +448,7 @@ def write_components( ), f"Expected {mgr.instr.name!r} to be the last uop" assert_no_pokes(managers) - if mgr.instr.name == "_SAVE_CURRENT_IP": + if mgr.instr.name == "_SAVE_RETURN_OFFSET": next_instr_is_set = True if cache_offset: out.emit(f"next_instr += {cache_offset};") diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 78b0c08..5ef5552 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1065,9 +1065,9 @@ class PyFramePtr: def _f_lasti(self): codeunit_p = gdb.lookup_type("_Py_CODEUNIT").pointer() - prev_instr = self._gdbval["prev_instr"] + instr_ptr = self._gdbval["instr_ptr"] first_instr = self._f_code().field("co_code_adaptive").cast(codeunit_p) - return int(prev_instr - first_instr) + return int(instr_ptr - first_instr) def is_shim(self): return self._f_special("owner", int) == FRAME_OWNED_BY_CSTACK -- cgit v0.12