diff options
| -rw-r--r-- | Include/internal/pycore_uop_ids.h | 7 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_metadata.h | 2 | ||||
| -rw-r--r-- | Python/bytecodes.c | 4 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 9 | ||||
| -rw-r--r-- | Python/optimizer.c | 6 | ||||
| -rw-r--r-- | Python/optimizer_analysis.c | 34 | ||||
| -rw-r--r-- | Python/pystate.c | 10 |
7 files changed, 66 insertions, 6 deletions
diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8ee90d7..a705658 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -230,9 +230,10 @@ extern "C" { #define _JUMP_TO_TOP 377 #define _SAVE_RETURN_OFFSET 378 #define _CHECK_VALIDITY 379 -#define _LOAD_CONST_INLINE_BORROW 380 -#define _INTERNAL_INCREMENT_OPT_COUNTER 381 -#define MAX_UOP_ID 381 +#define _LOAD_CONST_INLINE 380 +#define _LOAD_CONST_INLINE_BORROW 381 +#define _INTERNAL_INCREMENT_OPT_COUNTER 382 +#define MAX_UOP_ID 382 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 9bfb4f4..14d3382 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -202,6 +202,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_DEOPT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, + [_LOAD_CONST_INLINE] = 0, [_LOAD_CONST_INLINE_BORROW] = 0, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, }; @@ -329,6 +330,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7674ff8..18749ce 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4070,6 +4070,10 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } + op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = Py_NewRef(ptr); + } + op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = ptr; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2b4399b..241b905 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3390,6 +3390,15 @@ break; } + case _LOAD_CONST_INLINE: { + PyObject *value; + PyObject *ptr = (PyObject *)CURRENT_OPERAND(); + value = Py_NewRef(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _LOAD_CONST_INLINE_BORROW: { PyObject *value; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); diff --git a/Python/optimizer.c b/Python/optimizer.c index 1551a5e..4b6ed17 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -588,6 +588,9 @@ top: // Jump here after _PUSH_FRAME or likely branches ADD_TO_TRACE(uop, oparg, operand, target); if (uop == _POP_FRAME) { TRACE_STACK_POP(); + /* Set the operand to the code object returned to, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)code; DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -629,6 +632,9 @@ top: // Jump here after _PUSH_FRAME or likely branches instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); + /* Set the operand to the callee's code object, + * to assist optimization passes */ + trace[trace_length-1].operand = (uintptr_t)new_code; code = new_code; instr = _PyCode_CODE(code); DPRINTF(2, diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7db51f0..d122599 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -13,6 +13,39 @@ #include "pycore_optimizer.h" static void +peephole_opt(PyCodeObject *co, _PyUOpInstruction *buffer, int buffer_size) +{ + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch(opcode) { + case _LOAD_CONST: { + assert(co != NULL); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, buffer[pc].oparg); + buffer[pc].opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; + buffer[pc].operand = (uintptr_t)val; + break; + } + case _CHECK_PEP_523: + { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + buffer[pc].opcode = _NOP; + } + break; + } + case _PUSH_FRAME: + case _POP_FRAME: + co = (PyCodeObject *)buffer[pc].operand; + break; + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; + } + } +} + +static void remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { int last_set_ip = -1; @@ -59,6 +92,7 @@ _Py_uop_analyze_and_optimize( int curr_stacklen ) { + peephole_opt(co, buffer, buffer_size); remove_unneeded_uops(buffer, buffer_size); return 0; } diff --git a/Python/pystate.c b/Python/pystate.c index 23ddc78..548c77b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2608,11 +2608,15 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) { if (eval_frame == _PyEval_EvalFrameDefault) { - interp->eval_frame = NULL; + eval_frame = NULL; } - else { - interp->eval_frame = eval_frame; + if (eval_frame == interp->eval_frame) { + return; + } + if (eval_frame != NULL) { + _Py_Executors_InvalidateAll(interp); } + interp->eval_frame = eval_frame; } |