diff options
Diffstat (limited to 'Python/bytecodes.c')
| -rw-r--r-- | Python/bytecodes.c | 2020 |
1 files changed, 1233 insertions, 787 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c00de88..67061ac 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -218,41 +218,37 @@ dummy_func( }; inst(LOAD_FAST_CHECK, (-- value)) { - value = GETLOCAL(oparg); - if (value == NULL) { + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); ERROR_IF(1, error); } - Py_INCREF(value); + value = PyStackRef_DUP(value_s); } replicate(8) pure inst(LOAD_FAST, (-- value)) { - value = GETLOCAL(oparg); - assert(value != NULL); - Py_INCREF(value); + assert(PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)) != NULL); + value = PyStackRef_DUP(GETLOCAL(oparg)); } inst(LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value - GETLOCAL(oparg) = NULL; + GETLOCAL(oparg) = PyStackRef_NULL; } inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) { uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; - value1 = GETLOCAL(oparg1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } pure inst(LOAD_CONST, (-- value)) { - value = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(value); + value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } replicate(8) inst(STORE_FAST, (value --)) { @@ -267,8 +263,7 @@ dummy_func( uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); - value2 = GETLOCAL(oparg2); - Py_INCREF(value2); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); } inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { @@ -283,7 +278,7 @@ dummy_func( } pure inst(PUSH_NULL, (-- res)) { - res = NULL; + res = PyStackRef_NULL; } macro(END_FOR) = POP_TOP; @@ -291,8 +286,8 @@ dummy_func( tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + if (PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver))) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { ERROR_NO_POP(); } } @@ -300,27 +295,31 @@ dummy_func( } pure inst(END_SEND, (receiver, value -- value)) { - Py_DECREF(receiver); + (void)receiver; + PyStackRef_CLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { - if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value))) { ERROR_NO_POP(); } } - Py_DECREF(receiver); + PyStackRef_CLOSE(receiver); } inst(UNARY_NEGATIVE, (value -- res)) { - res = PyNumber_Negative(value); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure inst(UNARY_NOT, (value -- res)) { - assert(PyBool_Check(value)); - res = Py_IsFalse(value) ? Py_True : Py_False; + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); + res = PyStackRef_Is(value, PyStackRef_False) + ? PyStackRef_True : PyStackRef_False; } family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = { @@ -345,63 +344,66 @@ dummy_func( } op(_TO_BOOL, (value -- res)) { - int err = PyObject_IsTrue(value); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); ERROR_IF(err < 0, error); - res = err ? Py_True : Py_False; + res = err ? PyStackRef_True : PyStackRef_False; } macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - EXIT_IF(!PyBool_Check(value)); + EXIT_IF(!PyBool_Check(PyStackRef_AsPyObjectBorrow(value))); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyLong_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyList_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - res = Py_SIZE(value) ? Py_True : Py_False; + res = Py_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; DECREF_INPUTS(); } inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - EXIT_IF(!Py_IsNone(value)); + EXIT_IF(!PyStackRef_Is(value, PyStackRef_None)); STAT_INC(TO_BOOL, hit); - res = Py_False; + res = PyStackRef_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - EXIT_IF(!PyUnicode_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyUnicode_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); - if (value == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value)); - res = Py_False; + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } else { - assert(Py_SIZE(value)); + assert(Py_SIZE(value_o)); DECREF_INPUTS(); - res = Py_True; + res = PyStackRef_True; } } op(_REPLACE_WITH_TRUE, (value -- res)) { - Py_DECREF(value); - res = Py_True; + DECREF_INPUTS(); + res = PyStackRef_True; } macro(TO_BOOL_ALWAYS_TRUE) = @@ -410,9 +412,10 @@ dummy_func( _REPLACE_WITH_TRUE; inst(UNARY_INVERT, (value -- res)) { - res = PyNumber_Invert(value); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = { @@ -427,40 +430,56 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - EXIT_IF(!PyLong_CheckExact(left)); - EXIT_IF(!PyLong_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckExact(right_o)); } op(_GUARD_NOS_INT, (left, unused -- left, unused)) { - EXIT_IF(!PyLong_CheckExact(left)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyLong_CheckExact(left_o)); } op(_GUARD_TOS_INT, (value -- value)) { - EXIT_IF(!PyLong_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - ERROR_IF(res == NULL, error); + PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free);; + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_INT) = @@ -471,40 +490,59 @@ dummy_func( _GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - EXIT_IF(!PyFloat_CheckExact(left)); - EXIT_IF(!PyFloat_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyFloat_CheckExact(left_o)); + EXIT_IF(!PyFloat_CheckExact(right_o)); } op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { - EXIT_IF(!PyFloat_CheckExact(left)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyFloat_CheckExact(left_o)); } op(_GUARD_TOS_FLOAT, (value -- value)) { - EXIT_IF(!PyFloat_CheckExact(value)); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyFloat_CheckExact(value_o)); } pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; - DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *res_o; + DECREF_INPUTS_AND_REUSE_FLOAT(left_o, right_o, dres, res_o); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_FLOAT) = @@ -515,16 +553,23 @@ dummy_func( _GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - EXIT_IF(!PyUnicode_CheckExact(left)); - EXIT_IF(!PyUnicode_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + EXIT_IF(!PyUnicode_CheckExact(left_o)); + EXIT_IF(!PyUnicode_CheckExact(right_o)); } pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(BINARY_OP, hit); - res = PyUnicode_Concat(left, right); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_ADD_UNICODE) = @@ -537,9 +582,12 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(next_instr->op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(next_instr->op.arg); - DEOPT_IF(*target_local != left); + _PyStackRef *target_local = &GETLOCAL(next_instr->op.arg); + DEOPT_IF(!PyStackRef_Is(*target_local, left)); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -552,11 +600,13 @@ dummy_func( * only the locals reference, so PyUnicode_Append knows * that the string is safe to mutate. */ - assert(Py_REFCNT(left) >= 2); - _Py_DECREF_NO_DEALLOC(left); - PyUnicode_Append(target_local, right); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - ERROR_IF(*target_local == NULL, error); + assert(Py_REFCNT(left_o) >= 2); + _Py_DECREF_NO_DEALLOC(left_o); + PyObject *temp = PyStackRef_AsPyObjectBorrow(*target_local); + PyUnicode_Append(&temp, right_o); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + ERROR_IF(PyStackRef_IsNull(*target_local), error); // The STORE_FAST is already done. assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); @@ -586,44 +636,55 @@ dummy_func( } op(_BINARY_SUBSCR, (container, sub -- res)) { - res = PyObject_GetItem(container, sub); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + + PyObject *res_o = PyObject_GetItem(container_o, sub_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; inst(BINARY_SLICE, (container, start, stop -- res)) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + PyObject *res_o; // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { - res = NULL; + res_o = NULL; } else { - res = PyObject_GetItem(container, slice); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); } - Py_DECREF(container); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(container); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(STORE_SLICE, (v, container, start, stop -- )) { - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); int err; if (slice == NULL) { err = 1; } else { - err = PyObject_SetItem(container, slice, v); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectSteal(v)); Py_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); ERROR_IF(err, error); } - inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { + inst(BINARY_SUBSCR_LIST_INT, (unused/1, list_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -632,14 +693,18 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { + inst(BINARY_SUBSCR_STR_INT, (unused/1, str_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyUnicode_CheckExact(str)); DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -649,12 +714,16 @@ dummy_func( Py_UCS4 c = PyUnicode_READ_CHAR(str, index); DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + PyStackRef_CLOSE(str_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { + inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyTuple_CheckExact(tuple)); @@ -663,25 +732,33 @@ dummy_func( Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + Py_INCREF(res_o); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + PyStackRef_CLOSE(tuple_st); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { + inst(BINARY_SUBSCR_DICT, (unused/1, dict_st, sub_st -- res)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); + PyObject *res_o; + int rc = PyDict_GetItemRef(dict, sub, &res_o); if (rc == 0) { _PyErr_SetKeyError(sub); } DECREF_INPUTS(); ERROR_IF(rc <= 0, error); // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { + inst(BINARY_SUBSCR_GETITEM, (unused/1, container_st, sub_st -- unused)) { + PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); @@ -699,18 +776,20 @@ dummy_func( Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); STACK_SHRINK(2); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + new_frame->localsplus[0] = container_st; + new_frame->localsplus[1] = sub_st; frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) { - ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error); + ERROR_IF(_PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)) < 0, error); } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { - int err = PySet_Add(set, v); + int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -734,14 +813,17 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; - inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -753,50 +835,59 @@ dummy_func( STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); + PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + PyStackRef_CLOSE(list_st); } - inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { + inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub_st -- )) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, PyStackRef_AsPyObjectSteal(value)); + PyStackRef_CLOSE(dict_st); ERROR_IF(err, error); } inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ - int err = PyObject_DelItem(container, sub); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); DECREF_INPUTS(); ERROR_IF(err, error); } inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); - res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { + inst(CALL_INTRINSIC_2, (value2_st, value1_st -- res)) { assert(oparg <= MAX_INTRINSIC_2); - res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: - cause = args[1]; + cause = PyStackRef_AsPyObjectSteal(args[1]); /* fall through */ case 1: - exc = args[0]; + exc = PyStackRef_AsPyObjectSteal(args[0]); /* fall through */ case 0: if (do_raise(tstate, exc, cause)) { @@ -820,7 +911,7 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return retval; + return PyStackRef_AsPyObjectSteal(retval); } // The stack effect here is ambiguous. @@ -848,7 +939,7 @@ dummy_func( inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); @@ -883,14 +974,16 @@ dummy_func( _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); - _PyFrame_StackPush(frame, retval); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(retval)); LOAD_IP(frame->return_offset); goto resume_frame; } inst(GET_AITER, (obj -- iter)) { unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; @@ -905,30 +998,33 @@ dummy_func( ERROR_IF(true, error); } - iter = (*getter)(obj); + iter_o = (*getter)(obj_o); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); ERROR_IF(true, error); } + iter = PyStackRef_FromPyObjectSteal(iter_o); } inst(GET_ANEXT, (aiter -- aiter, awaitable)) { unaryfunc getter = NULL; PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); + PyObject *awaitable_o; + PyObject *aiter_o = PyStackRef_AsPyObjectBorrow(aiter); + PyTypeObject *type = Py_TYPE(aiter_o); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { + if (PyAsyncGen_CheckExact(aiter_o)) { + awaitable_o = type->tp_as_async->am_anext(aiter_o); + if (awaitable_o == NULL) { ERROR_NO_POP(); } } else { @@ -937,7 +1033,7 @@ dummy_func( } if (getter != NULL) { - next_iter = (*getter)(aiter); + next_iter = (*getter)(aiter_o); if (next_iter == NULL) { ERROR_NO_POP(); } @@ -950,8 +1046,8 @@ dummy_func( ERROR_NO_POP(); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { + awaitable_o = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable_o == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " @@ -964,32 +1060,35 @@ dummy_func( Py_DECREF(next_iter); } } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } inst(GET_AWAITABLE, (iterable -- iter)) { - iter = _PyCoro_GetAwaitableIter(iterable); + PyObject *iter_o = _PyCoro_GetAwaitableIter(PyStackRef_AsPyObjectBorrow(iterable)); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + if (iter_o == NULL) { + _PyEval_FormatAwaitableError(tstate, + Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)), oparg); } DECREF_INPUTS(); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (iter_o != NULL && PyCoro_CheckExact(iter_o)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter_o); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ Py_DECREF(yf); - Py_CLEAR(iter); + Py_CLEAR(iter_o); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); /* The code below jumps to `error` if `iter` is NULL. */ } } - ERROR_IF(iter == NULL, error); + ERROR_IF(iter_o == NULL, error); + iter = PyStackRef_FromPyObjectSteal(iter_o); } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { @@ -1009,12 +1108,15 @@ dummy_func( } op(_SEND, (receiver, v -- receiver, retval)) { + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + + PyObject *retval_o; assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); @@ -1025,33 +1127,36 @@ dummy_func( frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); + if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) { + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); } else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); } - if (retval == NULL) { + if (retval_o == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, this_instr); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); + if (_PyGen_FetchStopIterationValue(&retval_o) == 0) { + assert(retval_o != NULL); JUMPBY(oparg); } else { ERROR_NO_POP(); } } - Py_DECREF(v); + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } macro(SEND) = _SPECIALIZE_SEND + _SEND; inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { DEOPT_IF(tstate->interp->eval_frame); - PyGenObject *gen = (PyGenObject *)receiver; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); @@ -1076,7 +1181,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + frame, this_instr, PyStackRef_AsPyObjectBorrow(retval)); if (err) ERROR_NO_POP(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1129,13 +1234,17 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + Py_XSETREF(exc_info->exc_value, + PyStackRef_AsPyObjectBorrow(exc_value) == Py_None + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); } - tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { + tier1 inst(RERAISE, (values[oparg], exc_st -- values[oparg])) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = values[0]; + PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); if (PyLong_Check(lasti)) { frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti); assert(!_PyErr_Occurred(tstate)); @@ -1153,7 +1262,9 @@ dummy_func( goto exception_unwind; } - tier1 inst(END_ASYNC_FOR, (awaitable, exc -- )) { + tier1 inst(END_ASYNC_FOR, (awaitable_st, exc_st -- )) { + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { DECREF_INPUTS(); @@ -1166,13 +1277,15 @@ dummy_func( } } - tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); + if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { - value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); - none = Py_None; + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -1185,10 +1298,10 @@ dummy_func( // Keep in sync with _common_constants in opcode.py switch(oparg) { case CONSTANT_ASSERTIONERROR: - value = PyExc_AssertionError; + value = PyStackRef_FromPyObjectImmortal(PyExc_AssertionError); break; case CONSTANT_NOTIMPLEMENTEDERROR: - value = PyExc_NotImplementedError; + value = PyStackRef_FromPyObjectImmortal(PyExc_NotImplementedError); break; default: Py_FatalError("bad LOAD_COMMON_CONSTANT oparg"); @@ -1196,12 +1309,14 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error); - if (bc == NULL) { + PyObject *bc_o; + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o) < 0, error); + if (bc_o == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); ERROR_IF(true, error); } + bc = PyStackRef_FromPyObjectSteal(bc_o); } inst(STORE_NAME, (v -- )) { @@ -1215,9 +1330,9 @@ dummy_func( ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1262,8 +1377,8 @@ dummy_func( } op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + _PyStackRef *top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1272,40 +1387,43 @@ dummy_func( inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { assert(oparg == 2); - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2); STAT_INC(UNPACK_SEQUENCE, hit); - val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq)); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyTuple_CheckExact(seq_o)); + DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq)); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + DEOPT_IF(!PyList_CheckExact(seq_o)); + DEOPT_IF(PyList_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq); + PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { - *values++ = Py_NewRef(items[i]); + *values++ = PyStackRef_FromPyObjectNew(items[i]); } DECREF_INPUTS(); } inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject **top = stack_pointer + totalargs - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + _PyStackRef *top = stack_pointer + totalargs - 1; + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1331,7 +1449,8 @@ dummy_func( op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectSteal(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1340,14 +1459,14 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_DelAttr(owner, name); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyDict_SetItem(GLOBALS(), name, v); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1367,28 +1486,29 @@ dummy_func( } inst(LOAD_LOCALS, ( -- locals)) { - locals = LOCALS(); - if (locals == NULL) { + PyObject *l = LOCALS(); + if (l == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); ERROR_IF(true, error); } - Py_INCREF(locals); + locals = PyStackRef_FromPyObjectNew(l);; } inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + PyObject *v_o; + if (PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { + if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (v == NULL) { + if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1401,11 +1521,11 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v_o) < 0, error); + if (v_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0, error); + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1415,9 +1535,11 @@ dummy_func( } } DECREF_INPUTS(); + v = PyStackRef_FromPyObjectSteal(v_o); } inst(LOAD_NAME, (-- v)) { + PyObject *v_o; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1425,18 +1547,18 @@ dummy_func( ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (v_o == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v_o) < 0) { ERROR_NO_POP(); } - if (v == NULL) { + if (v_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1444,6 +1566,7 @@ dummy_func( } } } + v = PyStackRef_FromPyObjectSteal(v_o); } family(LOAD_GLOBAL, INLINE_CACHE_ENTRIES_LOAD_GLOBAL) = { @@ -1466,13 +1589,14 @@ dummy_func( op(_LOAD_GLOBAL, ( -- res, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *res_o; if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + res_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - if (res == NULL) { + if (res_o == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1485,11 +1609,11 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res_o) < 0, error); + if (res_o == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); - if (res == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res_o) < 0, error); + if (res_o == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1497,7 +1621,8 @@ dummy_func( } } } - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } macro(LOAD_GLOBAL) = @@ -1524,21 +1649,23 @@ dummy_func( op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; - DEOPT_IF(res == NULL); - Py_INCREF(res); + PyObject *res_o = entries[index].me_value; + DEOPT_IF(res_o == NULL); + Py_INCREF(res_o); STAT_INC(LOAD_GLOBAL, hit); - null = NULL; + null = PyStackRef_NULL; + res = PyStackRef_FromPyObjectSteal(res_o); } macro(LOAD_GLOBAL_MODULE) = @@ -1554,30 +1681,30 @@ dummy_func( _LOAD_GLOBAL_BUILTINS; inst(DELETE_FAST, (--)) { - PyObject *v = GETLOCAL(oparg); - if (v == NULL) { + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); ERROR_IF(1, error); } - SETLOCAL(oparg, NULL); + SETLOCAL(oparg, PyStackRef_NULL); } inst(MAKE_CELL, (--)) { // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). - PyObject *initial = GETLOCAL(oparg); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { ERROR_NO_POP(); } - SETLOCAL(oparg, cell); + SETLOCAL(oparg, PyStackRef_FromPyObjectSteal(cell)); } inst(DELETE_DEREF, (--)) { - PyObject *cell = GETLOCAL(oparg); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); @@ -1588,37 +1715,42 @@ dummy_func( Py_DECREF(oldobj); } - inst(LOAD_FROM_DICT_OR_DEREF, (class_dict -- value)) { + inst(LOAD_FROM_DICT_OR_DEREF, (class_dict_st -- value)) { + PyObject *value_o; PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + if (PyMapping_GetOptionalItem(class_dict, name, &value_o) < 0) { ERROR_NO_POP(); } - if (!value) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_NO_POP(); } } - Py_DECREF(class_dict); + PyStackRef_CLOSE(class_dict_st); + value = PyStackRef_FromPyObjectSteal(value_o); } inst(LOAD_DEREF, ( -- value)) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - value = PyCell_GetRef(cell); - if (value == NULL) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *value_o = PyCell_GetRef(cell); + if (value_o == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); ERROR_IF(true, error); } + value = PyStackRef_FromPyObjectSteal(value_o); } inst(STORE_DEREF, (v --)) { - PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); - PyCell_SetTakeRef(cell, v); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); } inst(COPY_FREE_VARS, (--)) { @@ -1630,27 +1762,51 @@ dummy_func( int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = Py_NewRef(o); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } } inst(BUILD_STRING, (pieces[oparg] -- str)) { - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); DECREF_INPUTS(); - ERROR_IF(str == NULL, error); + ERROR_IF(str_o == NULL, error); + str = PyStackRef_FromPyObjectSteal(str_o); } inst(BUILD_TUPLE, (values[oparg] -- tup)) { - tup = _PyTuple_FromArraySteal(values, oparg); - ERROR_IF(tup == NULL, error); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + ERROR_IF(tup_o == NULL, error); + tup = PyStackRef_FromPyObjectSteal(tup_o); } inst(BUILD_LIST, (values[oparg] -- list)) { - list = _PyList_FromArraySteal(values, oparg); - ERROR_IF(list == NULL, error); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *list_o = _PyList_FromArraySteal(values_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + ERROR_IF(list_o == NULL, error); + list = PyStackRef_FromPyObjectSteal(list_o); } - inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) { + inst(LIST_EXTEND, (list_st, unused[oparg-1], iterable_st -- list_st, unused[oparg-1])) { + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1669,35 +1825,46 @@ dummy_func( } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { - int err = _PySet_Update(set, iterable); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); DECREF_INPUTS(); ERROR_IF(err < 0, error); } inst(BUILD_SET, (values[oparg] -- set)) { - set = PySet_New(NULL); - if (set == NULL) + PyObject *set_o = PySet_New(NULL); + if (set_o == NULL) { ERROR_NO_POP(); + } int err = 0; for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); + PyObject *item = PyStackRef_AsPyObjectSteal(values[i]); + if (err == 0) { + err = PySet_Add(set_o, item); + } Py_DECREF(item); } if (err != 0) { - Py_DECREF(set); + Py_DECREF(set_o); ERROR_IF(true, error); } + set = PyStackRef_FromPyObjectSteal(set_o); } inst(BUILD_MAP, (values[oparg*2] -- map)) { - map = _PyDict_FromItems( - values, 2, - values+1, 2, + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(SETUP_ANNOTATIONS, (--)) { @@ -1724,21 +1891,33 @@ dummy_func( } inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - assert(PyTuple_CheckExact(keys)); - assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + PyObject *keys_o = PyStackRef_AsPyObjectBorrow(keys); + + assert(PyTuple_CheckExact(keys_o)); + assert(PyTuple_GET_SIZE(keys_o) == (Py_ssize_t)oparg); + STACKREFS_TO_PYOBJECTS(values, oparg, values_o); + if (CONVERSION_FAILED(values_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *map_o = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys_o, 0), 1, + values_o, 1, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); DECREF_INPUTS(); - ERROR_IF(map == NULL, error); + ERROR_IF(map_o == NULL, error); + map = PyStackRef_FromPyObjectSteal(map_o); } inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { - if (PyDict_Update(dict, update) < 0) { + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (PyDict_Update(dict_o, update_o) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + Py_TYPE(update_o)->tp_name); } DECREF_INPUTS(); ERROR_IF(true, error); @@ -1747,19 +1926,24 @@ dummy_func( } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { - if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, callable, update); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + + if (_PyDict_MergeEx(dict_o, update_o, 2) < 0) { + _PyEval_FormatKwargsError(tstate, callable_o, update_o); DECREF_INPUTS(); ERROR_IF(true, error); } DECREF_INPUTS(); } - inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) { + inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); + ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value)) != 0, error); } inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { @@ -1774,12 +1958,12 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { + specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -1787,7 +1971,11 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { + tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -1818,26 +2006,35 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(super == NULL, error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = PyObject_GetAttr(super, name); + attr = PyStackRef_FromPyObjectSteal(PyObject_GetAttr(super, name)); Py_DECREF(super); - ERROR_IF(attr == NULL, error); - null = NULL; + ERROR_IF(PyStackRef_IsNull(attr), error); + null = PyStackRef_NULL; } macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); + attr_st = PyStackRef_FromPyObjectSteal(attr); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type); DEOPT_IF(!PyType_Check(class)); @@ -1845,20 +2042,22 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, + PyObject *attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + if (attr_o == NULL) { + PyStackRef_CLOSE(self_st); ERROR_IF(true, error); } if (method_found) { - self_or_null = self; // transfer ownership + self_or_null = self_st; // transfer ownership } else { - Py_DECREF(self); - self_or_null = NULL; + PyStackRef_CLOSE(self_st); + self_or_null = PyStackRef_NULL; } + + attr = PyStackRef_FromPyObjectSteal(attr_o); } family(LOAD_ATTR, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { @@ -1891,15 +2090,16 @@ dummy_func( op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + attr_o = NULL; + if (_PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr != NULL); // No errors on this branch + assert(attr_o != NULL); // No errors on this branch self_or_null = owner; // Transfer ownership } else { @@ -1910,16 +2110,17 @@ dummy_func( meth | NULL | arg1 | ... | argN */ DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); - self_or_null = NULL; + ERROR_IF(attr_o == NULL, error); + self_or_null = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); - ERROR_IF(attr == NULL, error); + ERROR_IF(attr_o == NULL, error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); } macro(LOAD_ATTR) = @@ -1928,23 +2129,26 @@ dummy_func( _LOAD_ATTR; op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { - PyTypeObject *tp = Py_TYPE(owner); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); EXIT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - attr = _PyObject_InlineValues(owner)->values[index]; - DEOPT_IF(attr == NULL); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o = _PyObject_InlineValues(owner_o)->values[index]; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -1956,22 +2160,25 @@ dummy_func( unused/5; // Skip over rest of cache op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) { - DEOPT_IF(!PyModule_CheckExact(owner)); - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!PyModule_CheckExact(owner_o)); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); DEOPT_IF(dict->ma_keys->dk_version != dict_version); } op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - attr = ep->me_value; - DEOPT_IF(attr == NULL); + PyObject *attr_o = ep->me_value; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -1982,30 +2189,36 @@ dummy_func( unused/5; op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject *attr_o; + + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); - attr = ep->me_value; + attr_o = ep->me_value; } - DEOPT_IF(attr == NULL); + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + Py_INCREF(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2017,12 +2230,14 @@ dummy_func( unused/5; split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { - char *addr = (char *)owner + index; - attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; + PyObject *attr_o = *(PyObject **)addr; + DEOPT_IF(attr_o == NULL); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr); - null = NULL; + null = PyStackRef_NULL; + attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -2033,17 +2248,19 @@ dummy_func( unused/5; op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { - DEOPT_IF(!PyType_Check(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + DEOPT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); + DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); } split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - null = NULL; + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2054,10 +2271,12 @@ dummy_func( _LOAD_ATTR_CLASS; inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(fget, &PyFunction_Type)); @@ -2078,9 +2297,11 @@ dummy_func( } inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); - PyTypeObject *cls = Py_TYPE(owner); + PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -2098,33 +2319,36 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - new_frame->localsplus[1] = Py_NewRef(name); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); } op(_GUARD_DORV_NO_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner)); - DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_GetManagedDict(owner_o)); + DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner) == NULL); - PyDictValues *values = _PyObject_InlineValues(owner); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyDictValues *values = _PyObject_InlineValues(owner_o); PyObject *old_value = values->values[index]; - values->values[index] = value; + values->values[index] = PyStackRef_AsPyObjectSteal(value); if (old_value == NULL) { _PyDictValues_AddToInsertionOrder(values, index); } else { Py_DECREF(old_value); } - - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = @@ -2134,8 +2358,9 @@ dummy_func( _STORE_ATTR_INSTANCE_VALUE; op(_STORE_ATTR_WITH_HINT, (hint/1, value, owner --)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2147,26 +2372,26 @@ dummy_func( DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name); old_value = ep->me_value; DEOPT_IF(old_value == NULL); - new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + ep->me_value = PyStackRef_AsPyObjectSteal(value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ - if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { + if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(PyStackRef_AsPyObjectBorrow(value))) { _PyObject_GC_TRACK(dict); } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_WITH_HINT) = @@ -2175,12 +2400,14 @@ dummy_func( _STORE_ATTR_WITH_HINT; op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { - char *addr = (char *)owner + index; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = value; + *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); Py_XDECREF(old_value); - Py_DECREF(owner); + PyStackRef_CLOSE(owner); } macro(STORE_ATTR_SLOT) = @@ -2207,15 +2434,21 @@ dummy_func( } op(_COMPARE_OP, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); ERROR_IF(res_bool < 0, error); - res = res_bool ? Py_True : Py_False; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); } } @@ -2231,52 +2464,67 @@ dummy_func( _GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR; op(_COMPARE_OP_FLOAT, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT op(_COMPARE_OP_INT, (left, right -- res)) { - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; + _Py_DECREF_SPECIALIZED(left_o, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right_o, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } // Similar to COMPARE_OP_FLOAT, but for ==, != only op(_COMPARE_OP_STR, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left, right); + int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(left_o, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } inst(IS_OP, (left, right -- b)) { - int res = Py_Is(left, right) ^ oparg; +#ifdef Py_GIL_DISABLED + // On free-threaded builds, objects are conditionally immortalized. + // So their bits don't always compare equally. + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; +#else + int res = PyStackRef_Is(left, right) ^ oparg; +#endif DECREF_INPUTS(); - b = res ? Py_True : Py_False; + b = res ? PyStackRef_True : PyStackRef_False; } family(CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP) = { @@ -2285,10 +2533,13 @@ dummy_func( }; op(_CONTAINS_OP, (left, right -- b)) { - int res = PySequence_Contains(right, left); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + int res = PySequence_Contains(right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { @@ -2306,68 +2557,86 @@ dummy_func( macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP; inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) { - DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! - int res = _PySet_Contains((PySetObject *)right, left); + int res = _PySet_Contains((PySetObject *)right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { - DEOPT_IF(!PyDict_CheckExact(right)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + DEOPT_IF(!PyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); - int res = PyDict_Contains(right, left); + int res = PyDict_Contains(right_o, left_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - b = (res ^ oparg) ? Py_True : Py_False; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) { + inst(CHECK_EG_MATCH, (exc_value_st, match_type_st -- rest, match)) { + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - match = NULL; - rest = NULL; + PyObject *match_o = NULL; + PyObject *rest_o = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match_o, &rest_o); DECREF_INPUTS(); ERROR_IF(res < 0, error); - assert((match == NULL) == (rest == NULL)); - ERROR_IF(match == NULL, error); + assert((match_o == NULL) == (rest_o == NULL)); + ERROR_IF(match_o == NULL, error); - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + if (!Py_IsNone(match_o)) { + PyErr_SetHandledException(match_o); } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); } inst(CHECK_EXC_MATCH, (left, right -- left, b)) { - assert(PyExceptionInstance_Check(left)); - if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + + assert(PyExceptionInstance_Check(left_o)); + if (_PyEval_CheckExceptTypeValid(tstate, right_o) < 0) { DECREF_INPUTS(); ERROR_IF(true, error); } - int res = PyErr_GivenExceptionMatches(left, right); + int res = PyErr_GivenExceptionMatches(left_o, right_o); DECREF_INPUTS(); - b = res ? Py_True : Py_False; + b = res ? PyStackRef_True : PyStackRef_False; } tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_name(tstate, frame, name, fromlist, level); + PyObject *res_o = import_name(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - res = import_from(tstate, from, name); - ERROR_IF(res == NULL, error); + PyObject *res_o = import_from(tstate, PyStackRef_AsPyObjectBorrow(from), name); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } tier1 inst(JUMP_FORWARD, (--)) { @@ -2446,8 +2715,8 @@ dummy_func( } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2455,8 +2724,8 @@ dummy_func( } replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif @@ -2464,11 +2733,11 @@ dummy_func( } op(_IS_NONE, (value -- b)) { - if (Py_IsNone(value)) { - b = Py_True; + if (PyStackRef_Is(value, PyStackRef_None)) { + b = PyStackRef_True; } else { - b = Py_False; + b = PyStackRef_False; DECREF_INPUTS(); } } @@ -2490,55 +2759,63 @@ dummy_func( JUMPBY(-oparg); } - inst(GET_LEN, (obj -- obj, len_o)) { + inst(GET_LEN, (obj -- obj, len)) { // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(obj); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); ERROR_IF(len_i < 0, error); - len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = PyLong_FromSsize_t(len_i); ERROR_IF(len_o == NULL, error); + len = PyStackRef_FromPyObjectSteal(len_o); } inst(MATCH_CLASS, (subject, type, names -- attrs)) { // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); DECREF_INPUTS(); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); // Success! + attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { ERROR_IF(_PyErr_Occurred(tstate), error); // Error! - attrs = Py_None; // Failure! + attrs = PyStackRef_None; // Failure! } } inst(MATCH_MAPPING, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_SEQUENCE, (subject -- subject, res)) { - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? Py_True : Py_False; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; } inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). - values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - ERROR_IF(values_or_none == NULL, error); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + ERROR_IF(values_or_none_o == NULL, error); + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - iter = PyObject_GetIter(iterable); + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable))); DECREF_INPUTS(); - ERROR_IF(iter == NULL, error); + ERROR_IF(PyStackRef_IsNull(iter), error); } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ - if (PyCoro_CheckExact(iterable)) { + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { /* `iterable` is a coroutine */ if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { /* and it is used in a 'yield from' expression of a @@ -2550,13 +2827,13 @@ dummy_func( } iter = iterable; } - else if (PyGen_CheckExact(iterable)) { + else if (PyGen_CheckExact(iterable_o)) { iter = iterable; } else { /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - if (iter == NULL) { + iter = PyStackRef_FromPyObjectSteal(PyObject_GetIter(iterable_o)); + if (PyStackRef_IsNull(iter)) { ERROR_NO_POP(); } DECREF_INPUTS(); @@ -2590,8 +2867,10 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + next = PyStackRef_NULL; if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); @@ -2602,19 +2881,21 @@ dummy_func( /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ JUMPBY(oparg + 2); DISPATCH(); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); @@ -2625,6 +2906,7 @@ dummy_func( /* The translator sets the deopt target just past the matching END_FOR */ DEOPT_IF(true); } + next = PyStackRef_FromPyObjectSteal(next_o); // Common case: no jump, leave it to the code generator } @@ -2632,10 +2914,11 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; - PyObject *iter = TOP(); + _PyStackRef iter_stackref = TOP(); + PyObject *iter = PyStackRef_AsPyObjectBorrow(iter_stackref); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { - PUSH(next); + PUSH(PyStackRef_FromPyObjectSteal(next)); target = next_instr; } else { @@ -2650,7 +2933,7 @@ dummy_func( assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); - Py_DECREF(iter); + PyStackRef_CLOSE(iter_stackref); /* Skip END_FOR and POP_TOP */ target = next_instr + oparg + 2; } @@ -2658,12 +2941,13 @@ dummy_func( } op(_ITER_CHECK_LIST, (iter -- iter)) { - EXIT_IF(Py_TYPE(iter) != &PyListIter_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type); } replaced op(_ITER_JUMP_LIST, (iter -- iter)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); PyListObject *seq = it->it_seq; if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { @@ -2674,7 +2958,7 @@ dummy_func( Py_DECREF(seq); } #endif - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2684,20 +2968,22 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; EXIT_IF(seq == NULL); EXIT_IF((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)); } op(_ITER_NEXT_LIST, (iter -- iter, next)) { - _PyListIterObject *it = (_PyListIterObject *)iter; - assert(Py_TYPE(iter) == &PyListIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyListIterObject *it = (_PyListIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); - next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_LIST) = @@ -2707,12 +2993,13 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - EXIT_IF(Py_TYPE(iter) != &PyTupleIter_Type); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type); } replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); PyTupleObject *seq = it->it_seq; if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { @@ -2720,7 +3007,7 @@ dummy_func( it->it_seq = NULL; Py_DECREF(seq); } - Py_DECREF(iter); + PyStackRef_CLOSE(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -2730,20 +3017,22 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; EXIT_IF(seq == NULL); EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); } op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { - _PyTupleIterObject *it = (_PyTupleIterObject *)iter; - assert(Py_TYPE(iter) == &PyTupleIter_Type); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + assert(Py_TYPE(iter_o) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); } macro(FOR_ITER_TUPLE) = @@ -2753,17 +3042,17 @@ dummy_func( _ITER_NEXT_TUPLE; op(_ITER_CHECK_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); } replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); + PyStackRef_CLOSE(iter); // Jump over END_FOR and POP_TOP instructions. JUMPBY(oparg + 2); DISPATCH(); @@ -2772,20 +3061,21 @@ dummy_func( // Only used by Tier 2 op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); EXIT_IF(r->len <= 0); } op(_ITER_NEXT_RANGE, (iter -- iter, next)) { - _PyRangeIterObject *r = (_PyRangeIterObject *)iter; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; - next = PyLong_FromLong(value); - ERROR_IF(next == NULL, error); + PyObject *res = PyLong_FromLong(value); + ERROR_IF(res == NULL, error); + next = PyStackRef_FromPyObjectSteal(res); } macro(FOR_ITER_RANGE) = @@ -2795,12 +3085,12 @@ dummy_func( _ITER_NEXT_RANGE; op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { - PyGenObject *gen = (PyGenObject *)iter; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, Py_None); + _PyFrame_StackPush(gen_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -2816,16 +3106,19 @@ dummy_func( inst(LOAD_SPECIAL, (owner -- attr, self_or_null)) { assert(oparg <= SPECIAL_MAX); + PyObject *owner_o = PyStackRef_AsPyObjectSteal(owner); PyObject *name = _Py_SpecialMethods[oparg].name; - attr = _PyObject_LookupSpecialMethod(owner, name, &self_or_null); - if (attr == NULL) { + PyObject *self_or_null_o; + attr = PyStackRef_FromPyObjectSteal(_PyObject_LookupSpecialMethod(owner_o, name, &self_or_null_o)); + if (PyStackRef_IsNull(attr)) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, _Py_SpecialMethods[oparg].error, - Py_TYPE(owner)->tp_name); + Py_TYPE(owner_o)->tp_name); } } - ERROR_IF(attr == NULL, error); + ERROR_IF(PyStackRef_IsNull(attr), error); + self_or_null = PyStackRef_FromPyObjectSteal(self_or_null_o); } inst(WITH_EXCEPT_START, (exit_func, exit_self, lasti, unused, val -- exit_func, exit_self, lasti, unused, val, res)) { @@ -2840,22 +3133,25 @@ dummy_func( */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } else { Py_DECREF(tb); } - assert(PyLong_Check(lasti)); + assert(PyLong_Check(PyStackRef_AsPyObjectBorrow(lasti))); (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[5] = {NULL, exit_self, exc, val, tb}; - int has_self = (exit_self != NULL); - res = PyObject_Vectorcall(exit_func, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - ERROR_IF(res == NULL, error); + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + res = PyStackRef_FromPyObjectSteal(PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL)); + ERROR_IF(PyStackRef_IsNull(res), error); } pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { @@ -2883,24 +3179,26 @@ dummy_func( }; inst(PUSH_EXC_INFO, (new_exc -- prev_exc, new_exc)) { + _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { - prev_exc = exc_info->exc_value; + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { - prev_exc = Py_None; + prev_exc = PyStackRef_None; } - assert(PyExceptionInstance_Check(new_exc)); - exc_info->exc_value = Py_NewRef(new_exc); + assert(PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(new_exc))); + exc_info->exc_value = PyStackRef_AsPyObjectNew(new_exc); } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner)->valid); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { - PyTypeObject *owner_cls = Py_TYPE(owner); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } @@ -2910,8 +3208,8 @@ dummy_func( /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - attr = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2924,11 +3222,11 @@ dummy_func( op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2943,7 +3241,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) = @@ -2955,11 +3253,11 @@ dummy_func( op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - assert(Py_TYPE(owner)->tp_dictoffset == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); } macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) = @@ -2969,7 +3267,7 @@ dummy_func( _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { - char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); @@ -2980,7 +3278,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = Py_NewRef(descr); + attr = PyStackRef_FromPyObjectNew(descr); self = owner; } @@ -2992,11 +3290,11 @@ dummy_func( _LOAD_ATTR_METHOD_LAZY_DICT; inst(INSTRUMENTED_CALL, (unused/3 -- )) { - int is_meth = PEEK(oparg + 1) != NULL; + int is_meth = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 1)) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 2); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 2)); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(PEEK(total_args)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3034,7 +3332,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -3044,31 +3342,35 @@ dummy_func( // When calling Python, inline the call using DISPATCH_INLINED(). op(_CALL, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - else if (Py_TYPE(callable) == &PyMethod_Type) { + else if (Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; } // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -3082,33 +3384,40 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } op(_CHECK_PERIODIC, (--)) { @@ -3118,17 +3427,20 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - assert(Py_TYPE(callable) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, args, total_args, NULL ); // The frame has stolen all the arguments from the stack, @@ -3140,8 +3452,9 @@ dummy_func( } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - EXIT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(!PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; EXIT_IF(func->func_version != func_version); } @@ -3154,23 +3467,25 @@ dummy_func( _PUSH_FRAME; op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { - EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); - PyObject *func = ((PyMethodObject *)callable)->im_func; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable_o)->im_func; EXIT_IF(!PyFunction_Check(func)); EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); - EXIT_IF(null != NULL); + EXIT_IF(!PyStackRef_IsNull(null)); } op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { - assert(null == NULL); - assert(Py_TYPE(callable) == &PyMethod_Type); - self = ((PyMethodObject *)callable)->im_self; - Py_INCREF(self); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + + assert(PyStackRef_IsNull(null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL - method = ((PyMethodObject *)callable)->im_func; - assert(PyFunction_Check(method)); - Py_INCREF(method); - Py_DECREF(callable); + method = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyFunction_Check(PyStackRef_AsPyObjectBorrow(method))); + PyStackRef_CLOSE(callable); } macro(CALL_BOUND_METHOD_GENERAL) = @@ -3183,30 +3498,41 @@ dummy_func( _PUSH_FRAME; op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - EXIT_IF(PyFunction_Check(callable)); - EXIT_IF(Py_TYPE(callable) == &PyMethod_Type); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + EXIT_IF(PyFunction_Check(callable_o)); + EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_NON_PY_GENERAL) = @@ -3217,17 +3543,20 @@ dummy_func( _CHECK_PERIODIC; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - EXIT_IF(null != NULL); - EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); + EXIT_IF(!PyStackRef_IsNull(null)); + EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); - self = Py_NewRef(((PyMethodObject *)callable)->im_self); - stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS - func = Py_NewRef(((PyMethodObject *)callable)->im_func); - stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + stack_pointer[-1 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS + stack_pointer[-2 - oparg] = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); // This is used by CALL, upon deoptimization + self = stack_pointer[-1 - oparg]; + func = stack_pointer[-2 - oparg]; + PyStackRef_CLOSE(callable); + // self may be unused in tier 1, so silence warnings. + (void)self; } op(_CHECK_PEP_523, (--)) { @@ -3235,25 +3564,28 @@ dummy_func( } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - assert(PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - EXIT_IF(code->co_argcount != oparg + (self_or_null != NULL)); + EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { - PyFunctionObject *func = (PyFunctionObject *)callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); DEOPT_IF(tstate->py_recursion_remaining <= 1); } replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { - int has_self = (self_or_null != NULL); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable; + PyFunctionObject *func = (PyFunctionObject *)callable_o; new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self); - PyObject **first_non_self_local = new_frame->localsplus + has_self; + _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; @@ -3298,22 +3630,28 @@ dummy_func( _PUSH_FRAME; inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyType_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); - res = Py_NewRef(Py_TYPE(arg)); - Py_DECREF(arg); + res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); + PyStackRef_CLOSE(arg); } op(_CALL_STR_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); - res = PyObject_Str(arg); - Py_DECREF(arg); - ERROR_IF(res == NULL, error); + res = PyStackRef_FromPyObjectSteal(PyObject_Str(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } macro(CALL_STR_1) = @@ -3323,13 +3661,16 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_TUPLE_1, (callable, null, arg -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); - DEOPT_IF(null != NULL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type); + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); - res = PySequence_Tuple(arg); - Py_DECREF(arg); - ERROR_IF(res == NULL, error); + res = PyStackRef_FromPyObjectSteal(PySequence_Tuple(arg_o)); + PyStackRef_CLOSE(arg); + ERROR_IF(PyStackRef_IsNull(res), error); } macro(CALL_TUPLE_1) = @@ -3339,18 +3680,19 @@ dummy_func( _CHECK_PERIODIC; inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; - DEOPT_IF(null != NULL); - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyStackRef_IsNull(null)); + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; DEOPT_IF(code->co_argcount != oparg+1); @@ -3360,17 +3702,17 @@ dummy_func( if (self == NULL) { ERROR_NO_POP(); } - Py_DECREF(tp); + PyStackRef_CLOSE(callable); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); - shim->localsplus[0] = self; + shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self); Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - init_frame->localsplus[0] = self; + init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self); for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } @@ -3391,31 +3733,40 @@ dummy_func( inst(EXIT_INIT_CHECK, (should_be_none -- )) { assert(STACK_LEVEL() == 2); - if (should_be_none != Py_None) { + if (!PyStackRef_Is(should_be_none, PyStackRef_None)) { PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); ERROR_NO_POP(); } } op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyType_Check(callable)); - PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(!PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(tp); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_CLASS) = @@ -3426,27 +3777,30 @@ dummy_func( op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - PyObject *arg = args[0]; + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(arg); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_O) = @@ -3457,28 +3811,37 @@ dummy_func( op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_FAST) = @@ -3489,27 +3852,38 @@ dummy_func( op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable)); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); + DEOPT_IF(!PyCFunction_CheckExact(callable_o)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyCFunction_GET_FUNCTION(callable_o); + + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) = @@ -3520,69 +3894,75 @@ dummy_func( inst(CALL_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* len(o) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len); + DEOPT_IF(callable_o != interp->callable_cache.len); STAT_INC(CALL, hit); - PyObject *arg = args[0]; + _PyStackRef arg_stackref = args[0]; + PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { ERROR_NO_POP(); } - res = PyLong_FromSsize_t(len_i); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + PyStackRef_CLOSE(callable); + PyStackRef_CLOSE(arg_stackref); + res = PyStackRef_FromPyObjectSteal(res_o); } inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* isinstance(o, o2) */ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance); + DEOPT_IF(callable_o != interp->callable_cache.isinstance); STAT_INC(CALL, hit); - PyObject *cls = args[1]; - PyObject *inst = args[0]; - int retval = PyObject_IsInstance(inst, cls); + _PyStackRef cls_stackref = args[1]; + _PyStackRef inst_stackref = args[0]; + int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); if (retval < 0) { ERROR_NO_POP(); } - res = PyBool_FromLong(retval); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res == NULL) { - GOTO_ERROR(error); - } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(inst_stackref); + PyStackRef_CLOSE(cls_stackref); + PyStackRef_CLOSE(callable); } // This is secretly a super-instruction tier1 inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- unused)) { assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append); - assert(self != NULL); - DEOPT_IF(!PyList_Check(self)); + DEOPT_IF(callable_o != interp->callable_cache.list_append); + assert(self_o != NULL); + DEOPT_IF(!PyList_Check(self_o)); STAT_INC(CALL, hit); - if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { + if (_PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + PyStackRef_CLOSE(self); + PyStackRef_CLOSE(callable); STACK_SHRINK(3); // Skip POP_TOP assert(next_instr->op.code == POP_TOP); @@ -3591,31 +3971,38 @@ dummy_func( } op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(total_args != 2); DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O); // CPython promises to check all non-vectorcall function calls. DEOPT_IF(tstate->c_recursion_remaining <= 0); - PyObject *arg = args[1]; - PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + _PyStackRef arg_stackref = args[1]; + _PyStackRef self_stackref = args[0]; + DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, arg); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(arg_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_O) = @@ -3625,31 +4012,41 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) = @@ -3660,16 +4057,19 @@ dummy_func( op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } DEOPT_IF(total_args != 1); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); DEOPT_IF(meth->ml_flags != METH_NOARGS); // CPython promises to check all non-vectorcall function calls. @@ -3677,12 +4077,13 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(self_stackref); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_NOARGS) = @@ -3692,30 +4093,41 @@ dummy_func( _CHECK_PERIODIC; op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; - if (self_or_null != NULL) { + if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL); - PyObject *self = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - Py_DECREF(callable); - ERROR_IF(res == NULL, error); + PyStackRef_CLOSE(callable); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(CALL_METHOD_DESCRIPTOR_FAST) = @@ -3725,11 +4137,11 @@ dummy_func( _CHECK_PERIODIC; inst(INSTRUMENTED_CALL_KW, ( -- )) { - int is_meth = PEEK(oparg + 2) != NULL; + int is_meth = !PyStackRef_IsNull(PEEK(oparg + 2)); int total_args = oparg + is_meth; - PyObject *function = PEEK(oparg + 3); + PyObject *function = PyStackRef_AsPyObjectBorrow(PEEK(oparg + 3)); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PyStackRef_AsPyObjectBorrow(PEEK(total_args + 1)); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); @@ -3738,35 +4150,40 @@ dummy_func( } inst(CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_or_null_o = PyStackRef_AsPyObjectBorrow(self_or_null); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: int total_args = oparg; - if (self_or_null != NULL) { + if (self_or_null_o != NULL) { args--; total_args++; } - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + if (self_or_null_o == NULL && Py_TYPE(callable_o) == &PyMethod_Type) { args--; total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + args[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + args[-1] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(callable); + callable_o = method; + callable = args[-1]; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && + if (Py_TYPE(callable_o) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, positional_args, kwnames + tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + args, positional_args, kwnames_o ); - Py_DECREF(kwnames); + PyStackRef_CLOSE(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, @@ -3779,34 +4196,41 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true, error); + } + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(args[0]); + if (res_o == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); + frame, this_instr, callable_o, arg); if (err < 0) { - Py_CLEAR(res); + Py_CLEAR(res_o); } } } - Py_DECREF(kwnames); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + PyStackRef_CLOSE(kwnames); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + PyStackRef_CLOSE(args[i]); } - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); CHECK_EVAL_BREAKER(); } @@ -3814,7 +4238,11 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) { + inst(CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3826,7 +4254,9 @@ dummy_func( if (tuple == NULL) { ERROR_NO_POP(); } - Py_SETREF(callargs, tuple); + PyStackRef_CLOSE(callargs_st); + callargs_st = PyStackRef_FromPyObjectSteal(tuple); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); @@ -3837,10 +4267,10 @@ dummy_func( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); if (err) ERROR_NO_POP(); - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result == NULL) { + if (PyStackRef_IsNull(result)) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); @@ -3850,7 +4280,7 @@ dummy_func( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); if (err < 0) { - Py_CLEAR(result); + PyStackRef_CLEAR(result); } } } @@ -3865,7 +4295,7 @@ dummy_func( PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, + (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); @@ -3876,30 +4306,34 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } - result = PyObject_Call(func, callargs, kwargs); + result = PyStackRef_FromPyObjectSteal(PyObject_Call(func, callargs, kwargs)); } DECREF_INPUTS(); - assert(PEEK(2 + (oparg & 1)) == NULL); - ERROR_IF(result == NULL, error); + assert(PyStackRef_AsPyObjectBorrow(PEEK(2 + (oparg & 1))) == NULL); + ERROR_IF(PyStackRef_IsNull(result), error); CHECK_EVAL_BREAKER(); } - inst(MAKE_FUNCTION, (codeobj -- func)) { + inst(MAKE_FUNCTION, (codeobj_st -- func)) { + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + PyStackRef_CLOSE(codeobj_st); if (func_obj == NULL) { ERROR_NO_POP(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = (PyObject *)func_obj; + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); } - inst(SET_FUNCTION_ATTRIBUTE, (attr, func -- func)) { + inst(SET_FUNCTION_ATTRIBUTE, (attr_st, func_st -- func_st)) { + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *attr = PyStackRef_AsPyObjectBorrow(attr_st); + assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -3947,7 +4381,7 @@ dummy_func( gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - res = (PyObject *)gen; + res = PyStackRef_FromPyObjectSteal((PyObject *)gen); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; @@ -3957,27 +4391,34 @@ dummy_func( } inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { - slice = PySlice_New(start, stop, step); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); - ERROR_IF(slice == NULL, error); + ERROR_IF(slice_o == NULL, error); + slice = PyStackRef_FromPyObjectSteal(slice_o); } inst(CONVERT_VALUE, (value -- result)) { conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; - result = conv_fn(value); - Py_DECREF(value); - ERROR_IF(result == NULL, error); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + ERROR_IF(result_o == NULL, error); + result = PyStackRef_FromPyObjectSteal(result_o); } inst(FORMAT_SIMPLE, (value -- res)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); /* If value is a unicode object, then we know the result * of format(value) is value itself. */ - if (!PyUnicode_CheckExact(value)) { - res = PyObject_Format(value, NULL); - Py_DECREF(value); - ERROR_IF(res == NULL, error); + if (!PyUnicode_CheckExact(value_o)) { + res = PyStackRef_FromPyObjectSteal(PyObject_Format(value_o, NULL)); + PyStackRef_CLOSE(value); + ERROR_IF(PyStackRef_IsNull(res), error); } else { res = value; @@ -3985,15 +4426,16 @@ dummy_func( } inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { - res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); - ERROR_IF(res == NULL, error); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(fmt_spec); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { assert(oparg > 0); - top = Py_NewRef(bottom); + top = PyStackRef_DUP(bottom); } specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { @@ -4011,10 +4453,14 @@ dummy_func( } op(_BINARY_OP, (lhs, rhs -- res)) { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(res_o == NULL, error); + res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; @@ -4047,9 +4493,9 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsTrue(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_True); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -4058,9 +4504,9 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { - PyObject *cond = POP(); - assert(PyBool_Check(cond)); - int flag = Py_IsFalse(cond); + _PyStackRef cond = POP(); + assert(PyBool_Check(PyStackRef_AsPyObjectBorrow(cond))); + int flag = PyStackRef_Is(cond, PyStackRef_False); int offset = flag * oparg; #if ENABLE_SPECIALIZATION this_instr[1].cache = (this_instr[1].cache << 1) | flag; @@ -4069,14 +4515,14 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { - PyObject *value = POP(); - int flag = Py_IsNone(value); + _PyStackRef value_stackref = POP(); + int flag = PyStackRef_Is(value_stackref, PyStackRef_None); int offset; if (flag) { offset = oparg; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = 0; } #if ENABLE_SPECIALIZATION @@ -4086,14 +4532,14 @@ dummy_func( } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { - PyObject *value = POP(); + _PyStackRef value_stackref = POP(); int offset; - int nflag = Py_IsNone(value); + int nflag = PyStackRef_Is(value_stackref, PyStackRef_None); if (nflag) { offset = 0; } else { - Py_DECREF(value); + PyStackRef_CLOSE(value_stackref); offset = oparg; } #if ENABLE_SPECIALIZATION @@ -4124,28 +4570,28 @@ dummy_func( op (_GUARD_IS_TRUE_POP, (flag -- )) { SYNC_SP(); - EXIT_IF(!Py_IsTrue(flag)); - assert(Py_IsTrue(flag)); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_True)); + assert(PyStackRef_Is(flag, PyStackRef_True)); } op (_GUARD_IS_FALSE_POP, (flag -- )) { SYNC_SP(); - EXIT_IF(!Py_IsFalse(flag)); - assert(Py_IsFalse(flag)); + EXIT_IF(!PyStackRef_Is(flag, PyStackRef_False)); + assert(PyStackRef_Is(flag, PyStackRef_False)); } op (_GUARD_IS_NONE_POP, (val -- )) { SYNC_SP(); - if (!Py_IsNone(val)) { - Py_DECREF(val); + if (!PyStackRef_Is(val, PyStackRef_None)) { + PyStackRef_CLOSE(val); EXIT_IF(1); } } op (_GUARD_IS_NOT_NONE_POP, (val -- )) { SYNC_SP(); - EXIT_IF(Py_IsNone(val)); - Py_DECREF(val); + EXIT_IF(PyStackRef_Is(val, PyStackRef_None)); + PyStackRef_CLOSE(val); } op(_JUMP_TO_TOP, (--)) { @@ -4180,26 +4626,26 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { - value = Py_NewRef(ptr); + value = PyStackRef_FromPyObjectNew(ptr); } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = ptr; + value = PyStackRef_FromPyObjectImmortal(ptr); } tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - Py_DECREF(pop); - value = ptr; + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectImmortal(ptr); } tier2 pure op(_LOAD_CONST_INLINE_WITH_NULL, (ptr/4 -- value, null)) { - value = Py_NewRef(ptr); - null = NULL; + value = PyStackRef_FromPyObjectNew(ptr); + null = PyStackRef_NULL; } tier2 pure op(_LOAD_CONST_INLINE_BORROW_WITH_NULL, (ptr/4 -- value, null)) { - value = ptr; - null = NULL; + value = PyStackRef_FromPyObjectImmortal(ptr); + null = PyStackRef_NULL; } tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { @@ -4209,7 +4655,7 @@ dummy_func( /* Internal -- for testing executors */ op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt; + _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); exe->count++; } |