diff options
author | Mark Shannon <mark@hotpy.org> | 2025-03-26 15:21:35 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-03-26 15:21:35 (GMT) |
commit | 1b8bb1ed0c4243796af531a35de982bc4f028215 (patch) | |
tree | 830d0586e0c7b5e65b03b40979794b0ecd99cfe5 /Python | |
parent | b9ca438daab6a4e2513ab38171e94c75da03d6e3 (diff) | |
download | cpython-1b8bb1ed0c4243796af531a35de982bc4f028215.zip cpython-1b8bb1ed0c4243796af531a35de982bc4f028215.tar.gz cpython-1b8bb1ed0c4243796af531a35de982bc4f028215.tar.bz2 |
GH-131729: Code-gen better liveness analysis (GH-131732)
* Rename 'defined' attribute to 'in_local' to more accurately reflect how it is used
* Make death of variables explicit even for array variables.
* Convert in_memory from boolean to stack offset
* Don't apply liveness analysis to optimizer generated code
* Fix RETURN_VALUE in optimizer
Diffstat (limited to 'Python')
-rw-r--r-- | Python/bytecodes.c | 39 | ||||
-rw-r--r-- | Python/executor_cases.c.h | 93 | ||||
-rw-r--r-- | Python/generated_cases.c.h | 134 | ||||
-rw-r--r-- | Python/optimizer_bytecodes.c | 4 | ||||
-rw-r--r-- | Python/optimizer_cases.c.h | 89 |
5 files changed, 126 insertions, 233 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0db4da0..44e8ea2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1927,6 +1927,7 @@ dummy_func( PyStackRef_CLOSE(value); } } + DEAD(values); if (err) { Py_DECREF(set_o); ERROR_IF(true, error); @@ -3583,15 +3584,15 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) { (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); PyStackRef_CLOSE(temp); } } @@ -3618,6 +3619,9 @@ dummy_func( tstate, callable[0], locals, arguments, total_args, NULL, frame ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); // Manipulate stack directly since we leave using DISPATCH_INLINED(). SYNC_SP(); // The frame has stolen all the arguments from the stack, @@ -3950,10 +3954,10 @@ dummy_func( _CALL_TUPLE_1 + _CHECK_PERIODIC; - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], null[1], args[oparg] -- init[1], self[1], args[oparg])) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) { (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0])); + DEOPT_IF(!PyStackRef_IsNull(self_or_null[0])); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); @@ -3969,9 +3973,9 @@ dummy_func( if (self_o == NULL) { ERROR_NO_POP(); } - self[0] = PyStackRef_FromPyObjectSteal(self_o); + self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); + callable[0] = PyStackRef_FromPyObjectNew(init_func); PyStackRef_CLOSE(temp); } @@ -3982,10 +3986,11 @@ dummy_func( assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); - DEAD(init); - DEAD(self); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); + DEAD(init); + DEAD(self); + DEAD(args); SYNC_SP(); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -4187,9 +4192,9 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_ISINSTANCE, (unused/1, unused/2, callable[1], self_or_null[1], args[oparg] -- res)) { + inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) { /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; @@ -4420,15 +4425,15 @@ dummy_func( ERROR_IF(err, error); } - op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) { + op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- callable[1], self_or_null[1], args[oparg], kwnames_out)) { (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); PyStackRef_CLOSE(temp); } kwnames_out = kwnames_in; @@ -4458,6 +4463,9 @@ dummy_func( tstate, callable[0], locals, arguments, positional_args, kwnames_o, frame ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); PyStackRef_CLOSE(kwnames); // Sync stack explicitly since we leave using DISPATCH_INLINED(). SYNC_SP(); @@ -4525,6 +4533,9 @@ dummy_func( PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. + DEAD(args); + DEAD(self_or_null); + DEAD(callable); SYNC_SP(); ERROR_IF(temp == NULL, error); new_frame = temp; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2942680..43f9a81 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -635,7 +635,6 @@ stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; break; } @@ -1259,13 +1258,13 @@ list_st = res; stack_pointer[-2] = list_st; PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(sub_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; break; } @@ -1354,7 +1353,6 @@ stack_pointer[-1] = tuple_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; break; } @@ -2054,6 +2052,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = val0; break; } @@ -2077,13 +2076,11 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - stack_pointer += -1; + stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2115,13 +2112,11 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); - stack_pointer += -1; + stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2701,11 +2696,11 @@ } } if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -3002,13 +2997,12 @@ global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; @@ -3296,7 +3290,6 @@ stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; break; } @@ -3331,7 +3324,6 @@ stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; break; } @@ -3936,9 +3928,7 @@ stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = b; } - stack_pointer[-1] = b; break; } @@ -4120,10 +4110,8 @@ stack_pointer[-1] = iterable; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = iter; } } - stack_pointer[-1] = iter; break; } @@ -4667,23 +4655,18 @@ _PyStackRef *args; _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *func; - _PyStackRef *maybe_self; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - args = &stack_pointer[-oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5252,21 +5235,16 @@ case _CHECK_AND_ALLOCATE_OBJECT: { _PyStackRef *args; - _PyStackRef *null; + _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *init; - _PyStackRef *self; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - args = &stack_pointer[-oparg]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyStackRef_IsNull(null[0])) { + if (!PyStackRef_IsNull(self_or_null[0])) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -5296,9 +5274,9 @@ if (self_o == NULL) { JUMP_TO_ERROR(); } - self[0] = PyStackRef_FromPyObjectSteal(self_o); + self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); + callable[0] = PyStackRef_FromPyObjectNew(init_func); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5720,14 +5698,14 @@ case _CALL_ISINSTANCE: { _PyStackRef *args; _PyStackRef *self_or_null; - _PyStackRef *callable; + _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -5755,8 +5733,9 @@ res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = callable[0]; - callable[0] = res; + _PyStackRef tmp = callable; + callable = res; + stack_pointer[-2 - oparg] = callable; PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; @@ -5767,7 +5746,6 @@ self_or_null[0] = PyStackRef_NULL; PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -6147,25 +6125,20 @@ _PyStackRef *args; _PyStackRef *self_or_null; _PyStackRef *callable; - _PyStackRef *func; - _PyStackRef *maybe_self; _PyStackRef kwnames_out; oparg = CURRENT_OPARG(); kwnames_in = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = &stack_pointer[-2 - oparg]; callable = &stack_pointer[-3 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - args = &stack_pointer[-1 - oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6668,13 +6641,13 @@ lhs = res; stack_pointer[-2] = lhs; PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(rhs); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a2e6c47..493d99d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -70,13 +70,13 @@ lhs = res; stack_pointer[-2] = lhs; PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(rhs); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; } DISPATCH(); } @@ -708,13 +708,13 @@ list_st = res; stack_pointer[-2] = list_st; PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(sub_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; DISPATCH(); } @@ -834,7 +834,6 @@ stack_pointer[-1] = tuple_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; DISPATCH(); } @@ -1125,11 +1124,11 @@ } } if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -1271,8 +1270,6 @@ _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; _PyStackRef res; // _SPECIALIZE_CALL { @@ -1296,17 +1293,14 @@ // _MAYBE_EXPAND_METHOD { args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - args = &stack_pointer[-oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1315,8 +1309,6 @@ // _DO_CALL { args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); // oparg counts all of the args, but *not* self: int total_args = oparg; @@ -1458,7 +1450,7 @@ INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; - _PyStackRef *null; + _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef *init; _PyStackRef *self; @@ -1476,15 +1468,12 @@ // _CHECK_AND_ALLOCATE_OBJECT { args = &stack_pointer[-oparg]; - null = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - init = &stack_pointer[-2 - oparg]; - self = &stack_pointer[-1 - oparg]; - args = &stack_pointer[-oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); (void)args; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - if (!PyStackRef_IsNull(null[0])) { + if (!PyStackRef_IsNull(self_or_null[0])) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -1518,18 +1507,17 @@ if (self_o == NULL) { JUMP_TO_LABEL(error); } - self[0] = PyStackRef_FromPyObjectSteal(self_o); + self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; - init[0] = PyStackRef_FromPyObjectNew(init_func); + callable[0] = PyStackRef_FromPyObjectNew(init_func); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); } // _CREATE_INIT_FRAME { - args = &stack_pointer[-oparg]; - self = &stack_pointer[-1 - oparg]; - init = &stack_pointer[-2 - oparg]; + self = self_or_null; + init = callable; _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); @@ -2547,7 +2535,7 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef *callable; + _PyStackRef callable; _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef res; @@ -2555,9 +2543,9 @@ /* Skip 2 cache entries */ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null[0])) { @@ -2587,8 +2575,9 @@ res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = callable[0]; - callable[0] = res; + _PyStackRef tmp = callable; + callable = res; + stack_pointer[-2 - oparg] = callable; PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; @@ -2599,7 +2588,6 @@ self_or_null[0] = PyStackRef_NULL; PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -2622,8 +2610,6 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; _PyStackRef kwnames_out; _PyStackRef res; // _SPECIALIZE_CALL_KW @@ -2649,17 +2635,14 @@ { kwnames_in = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - args = &stack_pointer[-1 - oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2670,8 +2653,6 @@ { kwnames = kwnames_out; args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); // oparg counts all of the args, but *not* self: @@ -2696,14 +2677,12 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -3958,6 +3937,7 @@ // _INIT_CALL_PY_EXACT_ARGS { args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame); @@ -6027,10 +6007,8 @@ stack_pointer[-1] = iterable; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = iter; } } - stack_pointer[-1] = iter; DISPATCH(); } @@ -6121,17 +6099,14 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - args = &stack_pointer[-oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6139,9 +6114,8 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-2 - oparg]; + maybe_self = self_or_null; + func = callable; int is_meth = !PyStackRef_IsNull(maybe_self[0]); PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]); PyObject *arg0; @@ -6506,8 +6480,6 @@ _PyStackRef *self_or_null; _PyStackRef *args; _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; _PyStackRef kwnames_out; _PyStackRef kwnames; _PyStackRef res; @@ -6519,17 +6491,14 @@ args = &stack_pointer[-1 - oparg]; self_or_null = &stack_pointer[-2 - oparg]; callable = &stack_pointer[-3 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - args = &stack_pointer[-1 - oparg]; (void)args; if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); + self_or_null[0] = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); + callable[0] = PyStackRef_FromPyObjectNew(method); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6538,9 +6507,6 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; int is_meth = !PyStackRef_IsNull(self_or_null[0]); PyObject *arg; if (is_meth) { @@ -6591,14 +6557,12 @@ arguments, positional_args, kwnames_o, frame ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -7873,7 +7837,6 @@ stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { @@ -7942,7 +7905,6 @@ stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { @@ -8603,7 +8565,6 @@ stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9655,13 +9616,12 @@ global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; @@ -9992,7 +9952,6 @@ stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = b; } } // _POP_JUMP_IF_TRUE @@ -10036,7 +9995,6 @@ stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = b; } } // _POP_JUMP_IF_FALSE @@ -11611,7 +11569,6 @@ stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-1] = res; DISPATCH(); } @@ -11880,13 +11837,11 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); - stack_pointer += -1; + stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11922,13 +11877,11 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - stack_pointer += -1; + stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11970,6 +11923,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = val0; DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index cba8787..02463fd 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -717,6 +717,8 @@ dummy_func(void) { } op(_RETURN_VALUE, (retval -- res)) { + JitOptSymbol *temp = retval; + DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); @@ -736,7 +738,7 @@ dummy_func(void) { ctx->done = true; } RELOAD_STACK(); - res = retval; + res = temp; } op(_RETURN_GENERATOR, ( -- res)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a111917..106e7e3 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -276,16 +276,14 @@ { assert(PyLong_CheckExact(sym_get_const(ctx, left))); assert(PyLong_CheckExact(sym_get_const(ctx, right))); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -311,16 +309,14 @@ { assert(PyLong_CheckExact(sym_get_const(ctx, left))); assert(PyLong_CheckExact(sym_get_const(ctx, right))); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -346,16 +342,14 @@ { assert(PyLong_CheckExact(sym_get_const(ctx, left))); assert(PyLong_CheckExact(sym_get_const(ctx, right))); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -557,17 +551,15 @@ goto error; } res = sym_new_const(ctx, temp); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); } else { res = sym_new_type(ctx, &PyUnicode_Type); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } // _STORE_FAST: GETLOCAL(this_instr->operand0) = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -710,6 +702,7 @@ JitOptSymbol *retval; JitOptSymbol *res; retval = stack_pointer[-1]; + JitOptSymbol *temp = retval; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; @@ -727,7 +720,7 @@ // might be impossible, but bailing is still safe ctx->done = true; } - res = retval; + res = temp; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -1259,11 +1252,7 @@ res = sym_new_type(ctx, &PyBool_Type); } else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); res = _Py_uop_sym_new_not_null(ctx); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-2] = res; stack_pointer += -1; @@ -1290,8 +1279,6 @@ { assert(PyLong_CheckExact(sym_get_const(ctx, left))); assert(PyLong_CheckExact(sym_get_const(ctx, right))); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), sym_get_const(ctx, right), oparg >> 5); @@ -1302,8 +1289,8 @@ assert(_Py_IsImmortal(tmp)); REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); res = sym_new_const(ctx, tmp); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-2] = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(tmp); } @@ -1673,16 +1660,14 @@ _Py_UOpsAbstractFrame *new_frame; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } new_frame = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[0] = (JitOptSymbol *)new_frame; - stack_pointer += 1; + stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1787,8 +1772,6 @@ int argcount = oparg; PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; @@ -1806,8 +1789,8 @@ } else { new_frame = frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[0] = (JitOptSymbol *)new_frame; - stack_pointer += 1; + stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2008,18 +1991,7 @@ /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */ case _MAYBE_EXPAND_METHOD_KW: { - JitOptSymbol **func; - JitOptSymbol **maybe_self; - JitOptSymbol **args; JitOptSymbol *kwnames_out; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - args = &stack_pointer[-1 - oparg]; - func[0] = sym_new_not_null(ctx); - maybe_self[0] = sym_new_not_null(ctx); - for (int _i = oparg; --_i >= 0;) { - args[_i] = sym_new_not_null(ctx); - } kwnames_out = sym_new_not_null(ctx); stack_pointer[-1] = kwnames_out; break; @@ -2111,6 +2083,7 @@ // might be impossible, but bailing is still safe ctx->done = true; } + stack_pointer[-1] = res; break; } @@ -2269,11 +2242,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, value != Py_True); - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } sym_set_const(flag, Py_True); stack_pointer += -1; @@ -2287,11 +2256,7 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, value != Py_False); - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } sym_set_const(flag, Py_False); stack_pointer += -1; @@ -2305,23 +2270,17 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } else { if (sym_has_type(flag)) { assert(!sym_matches_type(flag, &_PyNone_Type)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, true); - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); } sym_set_const(flag, Py_None); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2331,22 +2290,16 @@ if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, Py_IsNone(value)); } else { if (sym_has_type(flag)) { assert(!sym_matches_type(flag, &_PyNone_Type)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); eliminate_pop_guard(this_instr, false); - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } |