From 6f8b0e781ccd5d17f00f91d1c4eddba4f3e9a8ed Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 17 Nov 2022 11:36:03 -0800 Subject: Add a macro for "inlining" new frames (GH-99490) --- Python/bytecodes.c | 52 ++++++++-------------------------------------- Python/ceval.c | 10 +++++++++ Python/generated_cases.c.h | 52 ++++++++-------------------------------------- 3 files changed, 28 insertions(+), 86 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1575b53..0b2c9e8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -467,13 +467,8 @@ dummy_func( for (int i = 2; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } // stack effect: (__0 -- ) @@ -1938,13 +1933,8 @@ dummy_func( for (int i = 1; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } // error: LOAD_ATTR has irregular stack effect @@ -1979,13 +1969,8 @@ dummy_func( for (int i = 2; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } // stack effect: (__0, __1 -- ) @@ -2685,18 +2670,14 @@ dummy_func( DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_SetStackPointer(frame, stack_pointer); frame->yield_offset = oparg; - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); - frame->prev_instr = next_instr - 1; _PyFrame_StackPush(gen_frame, Py_NewRef(Py_None)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - frame = cframe.current_frame = gen_frame; - goto start_frame; + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); + assert(_Py_OPCODE(*next_instr) == END_FOR); + DISPATCH_INLINED(gen_frame); } // stack effect: ( -- __0) @@ -2978,13 +2959,8 @@ dummy_func( if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ PyObject *res; @@ -3032,7 +3008,6 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func); - CALL_STAT_INC(inlined_py_calls); STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; @@ -3041,12 +3016,8 @@ dummy_func( new_frame->localsplus[i] = NULL; } STACK_SHRINK(2-is_meth); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - goto start_frame; + DISPATCH_INLINED(new_frame); } // stack effect: (__0, __array[oparg] -- ) @@ -3067,7 +3038,6 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func); - CALL_STAT_INC(inlined_py_calls); STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; @@ -3081,12 +3051,8 @@ dummy_func( new_frame->localsplus[i] = NULL; } STACK_SHRINK(2-is_meth); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - goto start_frame; + DISPATCH_INLINED(new_frame); } // stack effect: (__0, __array[oparg] -- ) diff --git a/Python/ceval.c b/Python/ceval.c index bff8b5c..d28fdeb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -712,6 +712,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH_GOTO(); \ } +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + frame->prev_instr = next_instr - 1; \ + (NEW_FRAME)->previous = frame; \ + frame = cframe.current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + goto start_frame; \ + } while (0) + #define CHECK_EVAL_BREAKER() \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ if (_Py_atomic_load_relaxed_int32(eval_breaker)) { \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b8bc66b..7864647 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -461,13 +461,8 @@ for (int i = 2; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } TARGET(LIST_APPEND) { @@ -1942,13 +1937,8 @@ for (int i = 1; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -1982,13 +1972,8 @@ for (int i = 2; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2692,18 +2677,14 @@ DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - _PyFrame_SetStackPointer(frame, stack_pointer); frame->yield_offset = oparg; - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); - frame->prev_instr = next_instr - 1; _PyFrame_StackPush(gen_frame, Py_NewRef(Py_None)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - frame = cframe.current_frame = gen_frame; - goto start_frame; + JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); + assert(_Py_OPCODE(*next_instr) == END_FOR); + DISPATCH_INLINED(gen_frame); } TARGET(BEFORE_ASYNC_WITH) { @@ -2983,13 +2964,8 @@ if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ PyObject *res; @@ -3038,7 +3014,6 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func); - CALL_STAT_INC(inlined_py_calls); STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; @@ -3047,12 +3022,8 @@ new_frame->localsplus[i] = NULL; } STACK_SHRINK(2-is_meth); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - goto start_frame; + DISPATCH_INLINED(new_frame); } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3072,7 +3043,6 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func); - CALL_STAT_INC(inlined_py_calls); STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; @@ -3086,12 +3056,8 @@ new_frame->localsplus[i] = NULL; } STACK_SHRINK(2-is_meth); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - frame = cframe.current_frame = new_frame; - goto start_frame; + DISPATCH_INLINED(new_frame); } TARGET(CALL_NO_KW_TYPE_1) { -- cgit v0.12