diff options
author | Ken Jin <kenjin@python.org> | 2025-02-06 15:21:57 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2025-02-06 15:21:57 (GMT) |
commit | cb640b659e14cb0a05767054f95a9d25787b472d (patch) | |
tree | ab534fdb77f1dfb58fd704a53dc472099dc4dd8f /Python/generated_cases.c.h | |
parent | 555dc50c811e3e9ebdc30a1d511cf48a32666d6f (diff) | |
download | cpython-cb640b659e14cb0a05767054f95a9d25787b472d.zip cpython-cb640b659e14cb0a05767054f95a9d25787b472d.tar.gz cpython-cb640b659e14cb0a05767054f95a9d25787b472d.tar.bz2 |
gh-128563: A new tail-calling interpreter (GH-128718)
Co-authored-by: Garrett Gu <garrettgu777@gmail.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Diffstat (limited to 'Python/generated_cases.c.h')
-rw-r--r-- | Python/generated_cases.c.h | 3232 |
1 files changed, 2703 insertions, 529 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5820147..7927fa7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,15 +8,22 @@ #endif #define TIER_ONE 1 +#ifndef Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif { +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -58,7 +65,7 @@ PyStackRef_CLOSE(lhs); PyStackRef_CLOSE(rhs); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -69,6 +76,13 @@ } TARGET(BINARY_OP_ADD_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); @@ -82,8 +96,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT @@ -98,7 +120,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -109,6 +131,13 @@ } TARGET(BINARY_OP_ADD_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); @@ -122,8 +151,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT @@ -137,7 +174,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -148,6 +185,13 @@ } TARGET(BINARY_OP_ADD_UNICODE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); @@ -161,8 +205,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE @@ -176,7 +228,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -187,8 +239,14 @@ } TARGET(BINARY_OP_EXTEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_EXTEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); @@ -209,7 +267,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(!res, BINARY_OP); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND @@ -234,6 +296,13 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); @@ -246,8 +315,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE @@ -264,7 +341,11 @@ next_oparg = CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); - DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -284,7 +365,7 @@ *target_local = PyStackRef_FromPyObjectSteal(temp); PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (PyStackRef_IsNull(*target_local)) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -299,6 +380,13 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); @@ -312,8 +400,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT @@ -328,7 +424,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -339,6 +435,13 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); @@ -352,8 +455,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT @@ -367,7 +478,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -378,6 +489,13 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); @@ -391,8 +509,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT @@ -407,7 +533,7 @@ ((PyFloatObject *)right_o)->ob_fval; PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -418,6 +544,13 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); @@ -431,8 +564,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT @@ -446,7 +587,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -457,6 +598,11 @@ } TARGET(BINARY_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -502,7 +648,7 @@ PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -513,6 +659,11 @@ } TARGET(BINARY_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); @@ -551,7 +702,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -562,6 +713,13 @@ } TARGET(BINARY_SUBSCR_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); @@ -574,7 +732,11 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -588,7 +750,7 @@ PyStackRef_CLOSE(dict_st); PyStackRef_CLOSE(sub_st); if (rc <= 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); @@ -599,6 +761,13 @@ } TARGET(BINARY_SUBSCR_GETITEM) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_GETITEM; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); @@ -610,22 +779,42 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } } // _BINARY_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_SUBSCR, hit); } @@ -658,6 +847,13 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_LIST_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); @@ -670,19 +866,39 @@ list_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); stack_pointer = _PyFrame_GetStackPointer(frame); - DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); #else - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); @@ -702,6 +918,13 @@ } TARGET(BINARY_SUBSCR_STR_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_STR_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); @@ -714,14 +937,34 @@ str_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyUnicode_CheckExact(str)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); @@ -738,6 +981,13 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BINARY_SUBSCR_TUPLE_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); @@ -750,12 +1000,28 @@ tuple_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } + if (!PyTuple_CheckExact(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR)); + JUMP_TO_PREDICTED(BINARY_SUBSCR); + } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); @@ -774,6 +1040,11 @@ } TARGET(BUILD_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_LIST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -782,7 +1053,7 @@ values = &stack_pointer[-oparg]; PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); if (list_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } list = PyStackRef_FromPyObjectSteal(list_o); stack_pointer[-oparg] = list; @@ -792,6 +1063,11 @@ } TARGET(BUILD_MAP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_MAP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -805,7 +1081,7 @@ } stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -820,7 +1096,7 @@ if (map_o == NULL) { stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectSteal(map_o); stack_pointer[-oparg*2] = map; @@ -830,6 +1106,11 @@ } TARGET(BUILD_SET) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SET; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -845,7 +1126,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } int err = 0; for (int i = 0; i < oparg; i++) { @@ -864,7 +1145,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectSteal(set_o); stack_pointer[-oparg] = set; @@ -874,6 +1155,11 @@ } TARGET(BUILD_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -890,7 +1176,7 @@ if (slice_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); stack_pointer[-oparg] = slice; @@ -900,6 +1186,11 @@ } TARGET(BUILD_STRING) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_STRING; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -913,7 +1204,7 @@ } stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -923,7 +1214,7 @@ if (str_o == NULL) { stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; @@ -933,6 +1224,11 @@ } TARGET(BUILD_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = BUILD_TUPLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -941,7 +1237,7 @@ values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } tup = PyStackRef_FromPyObjectSteal(tup_o); stack_pointer[-oparg] = tup; @@ -951,6 +1247,11 @@ } TARGET(CACHE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CACHE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -960,13 +1261,17 @@ } TARGET(CALL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -1041,7 +1346,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -1056,7 +1361,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -1097,7 +1402,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1113,7 +1418,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1126,8 +1431,14 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1141,7 +1452,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_AND_ALLOCATE_OBJECT { @@ -1152,23 +1467,39 @@ self = &stack_pointer[-1 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(!PyType_Check(callable_o), CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } self[0] = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable[0]; @@ -1198,7 +1529,7 @@ assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); - goto error; + JUMP_TO_LABEL(error); } init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1227,8 +1558,14 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1240,14 +1577,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL); + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -1267,9 +1616,17 @@ { uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -1277,15 +1634,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -1329,8 +1698,14 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -1342,7 +1717,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_METHOD_VERSION { @@ -1350,11 +1729,27 @@ callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _EXPAND_METHOD { @@ -1394,7 +1789,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -1426,6 +1821,13 @@ } TARGET(CALL_BUILTIN_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); @@ -1442,7 +1844,11 @@ self_or_null = &stack_pointer[-1 - oparg]; callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyType_Check(callable_o), CALL); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; @@ -1450,7 +1856,11 @@ arguments--; total_args++; } - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -1461,7 +1871,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -1475,7 +1885,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1491,7 +1901,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1504,6 +1914,13 @@ } TARGET(CALL_BUILTIN_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); @@ -1527,8 +1944,16 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); /* res = func(self, args, nargs) */ @@ -1541,7 +1966,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)( @@ -1559,7 +1984,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1575,7 +2000,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1588,6 +2013,13 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); @@ -1611,8 +2043,16 @@ arguments--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); /* res = func(self, arguments, nargs, kwnames) */ _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1629,7 +2069,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -1644,7 +2084,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1660,7 +2100,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -1673,6 +2113,13 @@ } TARGET(CALL_BUILTIN_O) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_BUILTIN_O; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); @@ -1695,11 +2142,27 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -1718,7 +2181,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1734,7 +2197,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1747,11 +2210,16 @@ } TARGET(CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_FUNCTION_EX; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -1777,13 +2245,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -1824,7 +2292,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -1873,7 +2341,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -1905,7 +2373,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -1921,7 +2389,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1934,6 +2402,11 @@ } TARGET(CALL_INTRINSIC_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_1; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -1946,7 +2419,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -1954,6 +2427,11 @@ } TARGET(CALL_INTRINSIC_2) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_INTRINSIC_2; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -1971,7 +2449,7 @@ PyStackRef_CLOSE(value2_st); PyStackRef_CLOSE(value1_st); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1981,6 +2459,13 @@ } TARGET(CALL_ISINSTANCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_ISINSTANCE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); @@ -2002,9 +2487,17 @@ arguments--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL); + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyStackRef cls_stackref = arguments[1]; _PyStackRef inst_stackref = arguments[0]; @@ -2012,7 +2505,7 @@ int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - goto error; + JUMP_TO_LABEL(error); } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2028,13 +2521,17 @@ } TARGET(CALL_KW) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - opcode = CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -2123,7 +2620,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -2140,7 +2637,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2182,7 +2679,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2193,8 +2690,14 @@ } TARGET(CALL_KW_BOUND_METHOD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2207,7 +2710,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CHECK_METHOD_VERSION_KW { @@ -2215,11 +2722,27 @@ callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } PyObject *func = ((PyMethodObject *)callable_o)->im_func; - DEOPT_IF(!PyFunction_Check(func), CALL_KW); - DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW); - DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW); + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null[0])) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _EXPAND_METHOD_KW { @@ -2269,7 +2792,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2301,10 +2824,16 @@ } TARGET(CALL_KW_NON_PY) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_NON_PY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef kwnames; @@ -2317,8 +2846,16 @@ { callable = &stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL_KW); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CALL_KW_NON_PY { @@ -2346,7 +2883,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -2371,7 +2908,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2387,7 +2924,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2400,8 +2937,14 @@ } TARGET(CALL_KW_PY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_KW_PY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2413,16 +2956,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL_KW); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _CHECK_FUNCTION_VERSION_KW { callable = &stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL_KW); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } } // _PY_FRAME_KW { @@ -2458,7 +3013,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -2490,6 +3045,13 @@ } TARGET(CALL_LEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); @@ -2510,9 +3072,17 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.len, CALL); + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyStackRef arg_stackref = args[0]; PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref); @@ -2520,7 +3090,7 @@ Py_ssize_t len_i = PyObject_Length(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2543,6 +3113,13 @@ } TARGET(CALL_LIST_APPEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_LIST_APPEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); @@ -2559,10 +3136,22 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o), CALL); - DEOPT_IF(!LOCK_OBJECT(self_o), CALL); + if (!PyList_Check(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); @@ -2577,7 +3166,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } #if TIER_ONE // Skip the following POP_TOP. This is done here in tier one, and @@ -2589,6 +3178,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); @@ -2613,11 +3209,23 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2629,7 +3237,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = @@ -2646,7 +3254,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2662,7 +3270,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2675,6 +3283,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); @@ -2698,12 +3313,24 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2715,7 +3342,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -2732,7 +3359,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2748,7 +3375,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2761,6 +3388,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); @@ -2783,16 +3417,36 @@ args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2810,7 +3464,7 @@ PyStackRef_CLOSE(callable[0]); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2826,7 +3480,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2839,6 +3493,13 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); @@ -2862,16 +3523,36 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } // CPython promises to check all non-vectorcall function calls. - DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL); + if (tstate->c_recursion_remaining <= 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } _PyStackRef arg_stackref = arguments[1]; _PyStackRef self_stackref = arguments[0]; - DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type), CALL); + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -2890,7 +3571,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2906,7 +3587,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2919,10 +3600,16 @@ } TARGET(CALL_NON_PY_GENERAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *self_or_null; @@ -2934,8 +3621,16 @@ { callable = &stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(PyFunction_Check(callable_o), CALL); - DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CALL_NON_PY_GENERAL { @@ -2961,7 +3656,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -2979,7 +3674,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -2995,7 +3690,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3008,8 +3703,14 @@ } TARGET(CALL_PY_EXACT_ARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3020,16 +3721,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_EXACT_ARGS { @@ -3038,15 +3751,27 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL); + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _INIT_CALL_PY_EXACT_ARGS { @@ -3090,8 +3815,14 @@ } TARGET(CALL_PY_GENERAL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_PY_GENERAL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3102,16 +3833,28 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _CHECK_FUNCTION_VERSION { callable = &stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - DEOPT_IF(!PyFunction_Check(callable_o), CALL); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } PyFunctionObject *func = (PyFunctionObject *)callable_o; - DEOPT_IF(func->func_version != func_version, CALL); + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } // _PY_FRAME_GENERAL { @@ -3137,7 +3880,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - goto error; + JUMP_TO_LABEL(error); } new_frame = temp; } @@ -3169,6 +3912,13 @@ } TARGET(CALL_STR_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_STR_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); @@ -3187,8 +3937,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); @@ -3199,7 +3957,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3215,7 +3973,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3228,6 +3986,13 @@ } TARGET(CALL_TUPLE_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TUPLE_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); @@ -3246,8 +4011,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); @@ -3258,7 +4031,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3274,7 +4047,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3287,6 +4060,13 @@ } TARGET(CALL_TYPE_1) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CALL_TYPE_1; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); @@ -3303,8 +4083,16 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); - DEOPT_IF(!PyStackRef_IsNull(null), CALL); - DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL); + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o))); stack_pointer[-3] = res; @@ -3317,6 +4105,11 @@ } TARGET(CHECK_EG_MATCH) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EG_MATCH; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -3334,7 +4127,7 @@ if (err < 0) { PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -3345,11 +4138,11 @@ PyStackRef_CLOSE(exc_value_st); PyStackRef_CLOSE(match_type_st); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (!Py_IsNone(match_o)) { stack_pointer += -2; @@ -3368,6 +4161,11 @@ } TARGET(CHECK_EXC_MATCH) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CHECK_EXC_MATCH; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -3384,7 +4182,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { PyStackRef_CLOSE(right); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -3396,8 +4194,14 @@ } TARGET(CLEANUP_THROW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CLEANUP_THROW; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; @@ -3409,7 +4213,9 @@ last_sent_val_st = stack_pointer[-2]; sub_iter_st = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #ifndef Py_TAIL_CALL_INTERP assert(throwflag); + #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); @@ -3427,7 +4233,7 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer[-3] = none; stack_pointer[-2] = value; @@ -3437,6 +4243,11 @@ } TARGET(COMPARE_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -3475,7 +4286,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } if (oparg & 16) { stack_pointer += -2; @@ -3485,7 +4296,7 @@ Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) { - goto error; + JUMP_TO_LABEL(error); } res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -3502,6 +4313,13 @@ } TARGET(COMPARE_OP_FLOAT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_FLOAT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); @@ -3515,8 +4333,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyFloat_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT @@ -3540,6 +4366,13 @@ } TARGET(COMPARE_OP_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); @@ -3553,16 +4386,32 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP); + if (!PyLong_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_INT { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!_PyLong_IsCompact((PyLongObject *)right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3582,6 +4431,13 @@ } TARGET(COMPARE_OP_STR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COMPARE_OP_STR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); @@ -3595,8 +4451,16 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP); + if (!PyUnicode_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + if (!PyUnicode_CheckExact(right_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } } /* Skip 1 cache entry */ // _COMPARE_OP_STR @@ -3621,6 +4485,11 @@ } TARGET(CONTAINS_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -3658,7 +4527,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -3669,6 +4538,13 @@ } TARGET(CONTAINS_OP_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); @@ -3681,7 +4557,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP); + if (!PyDict_CheckExact(right_o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -3689,7 +4569,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3699,6 +4579,13 @@ } TARGET(CONTAINS_OP_SET) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONTAINS_OP_SET; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); @@ -3711,7 +4598,11 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP); + if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3720,7 +4611,7 @@ PyStackRef_CLOSE(left); PyStackRef_CLOSE(right); if (res < 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = b; @@ -3730,6 +4621,11 @@ } TARGET(CONVERT_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = CONVERT_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -3748,7 +4644,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; @@ -3758,6 +4654,11 @@ } TARGET(COPY) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -3773,6 +4674,11 @@ } TARGET(COPY_FREE_VARS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = COPY_FREE_VARS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -3791,6 +4697,11 @@ } TARGET(DELETE_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -3802,7 +4713,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3810,6 +4721,11 @@ } TARGET(DELETE_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -3821,7 +4737,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); @@ -3830,6 +4746,11 @@ } TARGET(DELETE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -3841,7 +4762,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -3852,6 +4773,11 @@ } TARGET(DELETE_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -3861,19 +4787,24 @@ stack_pointer = _PyFrame_GetStackPointer(frame); // Can't use ERROR_IF here. if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -3885,7 +4816,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -3897,12 +4828,17 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DELETE_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DELETE_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -3918,7 +4854,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -3926,6 +4862,11 @@ } TARGET(DICT_MERGE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_MERGE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -3946,7 +4887,7 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3955,6 +4896,11 @@ } TARGET(DICT_UPDATE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = DICT_UPDATE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -3979,7 +4925,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(update); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } PyStackRef_CLOSE(update); stack_pointer += -1; @@ -3988,8 +4934,14 @@ } TARGET(END_ASYNC_FOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_ASYNC_FOR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; @@ -4012,7 +4964,7 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -4020,6 +4972,11 @@ } TARGET(END_FOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_FOR; + (void)(opcode); next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -4038,6 +4995,11 @@ } TARGET(END_SEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = END_SEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -4056,11 +5018,16 @@ } TARGET(ENTER_EXECUTOR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = ENTER_EXECUTOR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -4090,6 +5057,11 @@ } TARGET(EXIT_INIT_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXIT_INIT_CHECK; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -4102,7 +5074,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -4110,10 +5082,14 @@ } TARGET(EXTENDED_ARG) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = EXTENDED_ARG; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -4122,6 +5098,11 @@ } TARGET(FORMAT_SIMPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_SIMPLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -4141,7 +5122,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4157,6 +5138,11 @@ } TARGET(FORMAT_WITH_SPEC) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FORMAT_WITH_SPEC; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -4171,7 +5157,7 @@ PyStackRef_CLOSE(value); PyStackRef_CLOSE(fmt_spec); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4181,6 +5167,11 @@ } TARGET(FOR_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -4219,7 +5210,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -4243,6 +5234,13 @@ } TARGET(FOR_ITER_GEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_GEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); @@ -4253,14 +5251,26 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } STAT_INC(FOR_ITER, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_None); @@ -4291,6 +5301,13 @@ } TARGET(FOR_ITER_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); @@ -4301,7 +5318,11 @@ // _ITER_CHECK_LIST { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_LIST { @@ -4342,6 +5363,13 @@ } TARGET(FOR_ITER_RANGE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_RANGE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); @@ -4353,7 +5381,11 @@ { iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_RANGE { @@ -4376,7 +5408,7 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { - goto error; + JUMP_TO_LABEL(error); } next = PyStackRef_FromPyObjectSteal(res); } @@ -4387,6 +5419,13 @@ } TARGET(FOR_ITER_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = FOR_ITER_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); @@ -4397,7 +5436,11 @@ // _ITER_CHECK_TUPLE { iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER); + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } } // _ITER_JUMP_TUPLE { @@ -4435,6 +5478,11 @@ } TARGET(GET_AITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -4456,14 +5504,14 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(obj); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { @@ -4476,7 +5524,7 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4484,6 +5532,11 @@ } TARGET(GET_ANEXT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ANEXT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -4494,7 +5547,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -4504,6 +5557,11 @@ } TARGET(GET_AWAITABLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_AWAITABLE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -4515,7 +5573,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4523,6 +5581,11 @@ } TARGET(GET_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -4535,7 +5598,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[-1] = iter; @@ -4543,6 +5606,11 @@ } TARGET(GET_LEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_LEN; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -4554,11 +5622,11 @@ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - goto error; + JUMP_TO_LABEL(error); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; @@ -4568,6 +5636,11 @@ } TARGET(GET_YIELD_FROM_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = GET_YIELD_FROM_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -4586,7 +5659,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } iter = iterable; } @@ -4600,7 +5673,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); PyStackRef_CLOSE(iterable); @@ -4611,6 +5684,11 @@ } TARGET(IMPORT_FROM) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_FROM; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -4622,7 +5700,7 @@ PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -4632,6 +5710,11 @@ } TARGET(IMPORT_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IMPORT_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -4649,7 +5732,7 @@ PyStackRef_CLOSE(level); PyStackRef_CLOSE(fromlist); if (res_o == NULL) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -4659,11 +5742,16 @@ } TARGET(INSTRUMENTED_CALL) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -4716,7 +5804,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _DO_CALL @@ -4750,7 +5838,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } frame->return_offset = 4 ; DISPATCH_INLINED(new_frame); @@ -4765,7 +5853,7 @@ } stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -4806,7 +5894,7 @@ if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4822,7 +5910,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4835,11 +5923,16 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef kwargs_in; @@ -4865,13 +5958,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } kwargs_out = kwargs_in; stack_pointer += -2; @@ -4912,7 +6005,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -4961,7 +6054,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 1 == 1); frame->return_offset = 1; @@ -4993,7 +6086,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -5009,7 +6102,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5022,11 +6115,16 @@ } TARGET(INSTRUMENTED_CALL_KW) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; @@ -5063,7 +6161,7 @@ frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _MAYBE_EXPAND_METHOD_KW @@ -5125,7 +6223,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4 ; @@ -5142,7 +6240,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5184,7 +6282,7 @@ if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -5195,6 +6293,11 @@ } TARGET(INSTRUMENTED_END_FOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_FOR; + (void)(opcode); _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -5210,7 +6313,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } PyStackRef_CLOSE(value); @@ -5220,8 +6323,14 @@ } TARGET(INSTRUMENTED_END_SEND) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; @@ -5235,7 +6344,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } val = value; @@ -5249,8 +6358,14 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ @@ -5269,7 +6384,7 @@ int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (!matches) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, this_instr); @@ -5286,17 +6401,22 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -5308,8 +6428,14 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ @@ -5322,7 +6448,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5334,8 +6460,14 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); @@ -5343,12 +6475,17 @@ } TARGET(INSTRUMENTED_LINE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LINE; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5364,7 +6501,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - goto error; + JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -5382,11 +6519,16 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -5412,7 +6554,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -5447,7 +6589,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -5457,7 +6599,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -5473,9 +6615,15 @@ } TARGET(INSTRUMENTED_NOT_TAKEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr @@ -5484,9 +6632,15 @@ } TARGET(INSTRUMENTED_POP_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; @@ -5501,8 +6655,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ @@ -5517,8 +6677,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ @@ -5537,8 +6703,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ @@ -5555,8 +6727,14 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ @@ -5571,8 +6749,14 @@ } TARGET(INSTRUMENTED_RESUME) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RESUME; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE @@ -5585,7 +6769,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -5609,7 +6793,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -5626,7 +6810,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5638,7 +6822,7 @@ tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -5649,8 +6833,14 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; @@ -5665,7 +6855,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } // _RETURN_VALUE @@ -5694,8 +6884,14 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; @@ -5710,7 +6906,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -5761,6 +6957,11 @@ } TARGET(INTERPRETER_EXIT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = INTERPRETER_EXIT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -5780,6 +6981,11 @@ } TARGET(IS_OP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = IS_OP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -5799,6 +7005,11 @@ } TARGET(JUMP_BACKWARD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -5826,7 +7037,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5844,8 +7055,14 @@ } TARGET(JUMP_BACKWARD_JIT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); @@ -5859,7 +7076,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5891,7 +7108,7 @@ if (optimized <= 0) { this_instr[1].counter = restart_backoff_counter(counter); if (optimized < 0) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -5912,6 +7129,11 @@ } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -5926,6 +7148,11 @@ } TARGET(JUMP_BACKWARD_NO_JIT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -5940,7 +7167,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -5958,6 +7185,11 @@ } TARGET(JUMP_FORWARD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = JUMP_FORWARD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -5966,6 +7198,11 @@ } TARGET(LIST_APPEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_APPEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -5976,7 +7213,7 @@ int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -5984,6 +7221,11 @@ } TARGET(LIST_EXTEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LIST_EXTEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -6011,7 +7253,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } PyStackRef_CLOSE(iterable_st); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } assert(Py_IsNone(none_val)); PyStackRef_CLOSE(iterable_st); @@ -6021,6 +7263,11 @@ } TARGET(LOAD_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -6077,7 +7324,7 @@ */ PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } self_or_null[0] = PyStackRef_NULL; } @@ -6089,7 +7336,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -6101,8 +7348,14 @@ } TARGET(LOAD_ATTR_CLASS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6115,9 +7368,17 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -6140,8 +7401,14 @@ } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6154,16 +7421,28 @@ owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_TYPE_VERSION { uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_CLASS { @@ -6185,8 +7464,14 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6198,17 +7483,33 @@ PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( @@ -6222,8 +7523,14 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6237,14 +7544,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_INSTANCE_VALUE { @@ -6252,10 +7567,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6278,8 +7601,14 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6293,7 +7622,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -6301,7 +7634,11 @@ char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT @@ -6322,8 +7659,14 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6337,7 +7680,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -6359,8 +7706,14 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6374,14 +7727,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_KEYS_VERSION { @@ -6389,7 +7750,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -6410,8 +7775,14 @@ } TARGET(LOAD_ATTR_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_MODULE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6425,11 +7796,19 @@ owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } mod_keys = keys; } // _LOAD_ATTR_MODULE_FROM_KEYS @@ -6440,11 +7819,19 @@ PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); // Clear mod_keys from stack in case we need to deopt - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } #else Py_INCREF(attr_o); @@ -6468,8 +7855,14 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6482,7 +7875,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -6500,8 +7897,14 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6514,14 +7917,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_KEYS_VERSION { @@ -6529,7 +7940,11 @@ PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -6545,8 +7960,14 @@ } TARGET(LOAD_ATTR_PROPERTY) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6555,7 +7976,11 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _GUARD_TYPE_VERSION { @@ -6563,7 +7988,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -6573,10 +8002,26 @@ assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; - DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR); - DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR); - DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; @@ -6611,8 +8056,14 @@ } TARGET(LOAD_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_SLOT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6626,7 +8077,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _LOAD_ATTR_SLOT { @@ -6634,10 +8089,18 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - DEOPT_IF(!increfed, LOAD_ATTR); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif @@ -6657,8 +8120,14 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); @@ -6673,14 +8142,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } // _CHECK_ATTR_WITH_HINT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict_o == NULL, LOAD_ATTR); + if (dict_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } assert(PyDict_CheckExact((PyObject *)dict_o)); dict = dict_o; } @@ -6689,26 +8166,46 @@ uint16_t hint = read_u16(&this_instr[4].cache); PyObject *attr_o; if (!LOCK_OBJECT(dict)) { - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } if (hint >= (size_t)dict->ma_keys->dk_nentries) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } attr_o = ep->me_value; if (attr_o == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, LOAD_ATTR); + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } } STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); @@ -6728,6 +8225,11 @@ } TARGET(LOAD_BUILD_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_BUILD_CLASS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -6737,14 +8239,14 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -6754,6 +8256,11 @@ } TARGET(LOAD_COMMON_CONSTANT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -6776,6 +8283,11 @@ } TARGET(LOAD_CONST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -6809,6 +8321,11 @@ } TARGET(LOAD_CONST_IMMORTAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_IMMORTAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); @@ -6824,6 +8341,11 @@ } TARGET(LOAD_CONST_MORTAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_CONST_MORTAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST_MORTAL); @@ -6838,6 +8360,11 @@ } TARGET(LOAD_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -6848,7 +8375,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; @@ -6858,6 +8385,11 @@ } TARGET(LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -6871,6 +8403,11 @@ } TARGET(LOAD_FAST_AND_CLEAR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -6884,6 +8421,11 @@ } TARGET(LOAD_FAST_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_CHECK; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -6896,7 +8438,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -6906,6 +8448,11 @@ } TARGET(LOAD_FAST_LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -6923,6 +8470,11 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -6939,7 +8491,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -6948,7 +8500,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -6964,6 +8516,11 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -6977,7 +8534,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(mod_or_class_dict); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -6999,7 +8556,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } } else { @@ -7011,7 +8568,7 @@ int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { /* namespace 2: builtins */ @@ -7019,7 +8576,7 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7027,7 +8584,7 @@ tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } } } @@ -7040,6 +8597,11 @@ } TARGET(LOAD_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -7076,7 +8638,7 @@ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) { - goto error; + JUMP_TO_LABEL(error); } } // _PUSH_NULL_CONDITIONAL @@ -7090,8 +8652,14 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7103,18 +8671,34 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } assert(DK_IS_UNICODE(keys)); } // _GUARD_BUILTINS_VERSION_PUSH_KEYS { uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } builtins_keys = keys; assert(DK_IS_UNICODE(builtins_keys)); } @@ -7123,10 +8707,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -7145,8 +8737,14 @@ } TARGET(LOAD_GLOBAL_MODULE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); @@ -7158,9 +8756,17 @@ { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } globals_keys = keys; assert(DK_IS_UNICODE(globals_keys)); } @@ -7170,10 +8776,18 @@ uint16_t index = read_u16(&this_instr[4].cache); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - DEOPT_IF(res_o == NULL, LOAD_GLOBAL); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - DEOPT_IF(!increfed, LOAD_GLOBAL); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } #else Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); @@ -7192,6 +8806,11 @@ } TARGET(LOAD_LOCALS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_LOCALS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -7202,7 +8821,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -7212,6 +8831,11 @@ } TARGET(LOAD_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -7221,7 +8845,7 @@ PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -7231,6 +8855,11 @@ } TARGET(LOAD_SMALL_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SMALL_INT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -7245,6 +8874,11 @@ } TARGET(LOAD_SPECIAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SPECIAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -7269,7 +8903,7 @@ Py_TYPE(owner_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); self_or_null = self_or_null_o == NULL ? @@ -7282,13 +8916,17 @@ } TARGET(LOAD_SUPER_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -7330,7 +8968,7 @@ PyStackRef_CLOSE(global_super_st); PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } // we make no attempt to optimize here; specializations should @@ -7365,7 +9003,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (super == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); stack_pointer += -3; @@ -7375,7 +9013,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -7391,6 +9029,13 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); @@ -7407,8 +9052,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7418,7 +9071,7 @@ PyStackRef_CLOSE(class_st); PyStackRef_CLOSE(self_st); if (attr == NULL) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[-3] = attr_st; @@ -7428,6 +9081,13 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); @@ -7445,8 +9105,16 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7456,7 +9124,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } if (method_found) { self_or_null = self_st; // transfer ownership @@ -7481,6 +9149,11 @@ } TARGET(MAKE_CELL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_CELL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -7489,7 +9162,7 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); @@ -7500,6 +9173,11 @@ } TARGET(MAKE_FUNCTION) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAKE_FUNCTION; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -7517,7 +9195,7 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -7529,6 +9207,11 @@ } TARGET(MAP_ADD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MAP_ADD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -7550,7 +9233,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -7558,6 +9241,11 @@ } TARGET(MATCH_CLASS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_CLASS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -7586,7 +9274,7 @@ } else { if (_PyErr_Occurred(tstate)) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } // Error! attrs = PyStackRef_None; // Failure! @@ -7598,6 +9286,11 @@ } TARGET(MATCH_KEYS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_KEYS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -7612,7 +9305,7 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -7622,6 +9315,11 @@ } TARGET(MATCH_MAPPING) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_MAPPING; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -7637,6 +9335,11 @@ } TARGET(MATCH_SEQUENCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = MATCH_SEQUENCE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -7652,6 +9355,11 @@ } TARGET(NOP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -7659,6 +9367,11 @@ } TARGET(NOT_TAKEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = NOT_TAKEN; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -7666,6 +9379,11 @@ } TARGET(POP_EXCEPT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_EXCEPT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -7683,6 +9401,11 @@ } TARGET(POP_ITER) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_ITER; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -7695,8 +9418,14 @@ } TARGET(POP_JUMP_IF_FALSE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; @@ -7712,8 +9441,14 @@ } TARGET(POP_JUMP_IF_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; @@ -7745,8 +9480,14 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; @@ -7778,8 +9519,14 @@ } TARGET(POP_JUMP_IF_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; @@ -7795,6 +9542,11 @@ } TARGET(POP_TOP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = POP_TOP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -7807,6 +9559,11 @@ } TARGET(PUSH_EXC_INFO) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_EXC_INFO; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -7832,6 +9589,11 @@ } TARGET(PUSH_NULL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = PUSH_NULL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -7844,8 +9606,14 @@ } TARGET(RAISE_VARARGS) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RAISE_VARARGS; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; @@ -7864,14 +9632,20 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - goto error; + JUMP_TO_LABEL(error); } TARGET(RERAISE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RERAISE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); _PyStackRef *values; @@ -7897,7 +9671,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); Py_DECREF(exc); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -7910,10 +9684,15 @@ monitor_reraise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } TARGET(RESERVED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESERVED; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -7923,6 +9702,11 @@ } TARGET(RESUME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -7939,7 +9723,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); @@ -7963,7 +9747,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); @@ -7988,7 +9772,7 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - goto error; + JUMP_TO_LABEL(error); } } } @@ -7997,26 +9781,50 @@ } TARGET(RESUME_CHECK) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RESUME_CHECK; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - DEOPT_IF(eval_breaker != version, RESUME); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } #ifdef Py_GIL_DISABLED - DEOPT_IF(frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME); + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } #endif DISPATCH(); } TARGET(RETURN_GENERATOR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_GENERATOR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -8027,7 +9835,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { - goto error; + JUMP_TO_LABEL(error); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8052,6 +9860,11 @@ } TARGET(RETURN_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = RETURN_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -8080,6 +9893,11 @@ } TARGET(SEND) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -8159,7 +9977,7 @@ } else { PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1; @@ -8176,6 +9994,13 @@ } TARGET(SEND_GEN) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SEND_GEN; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); @@ -8187,15 +10012,27 @@ /* Skip 1 cache entry */ // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, SEND); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } STAT_INC(SEND, hit); gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, v); @@ -8228,6 +10065,11 @@ } TARGET(SETUP_ANNOTATIONS) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SETUP_ANNOTATIONS; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -8237,21 +10079,21 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } /* check if __annotations__ in locals()... */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - goto error; + JUMP_TO_LABEL(error); } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) { - goto error; + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), @@ -8259,7 +10101,7 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } } else { @@ -8271,6 +10113,11 @@ } TARGET(SET_ADD) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_ADD; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -8284,7 +10131,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8292,6 +10139,11 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -8316,6 +10168,11 @@ } TARGET(SET_UPDATE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SET_UPDATE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -8329,7 +10186,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(iterable); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8337,6 +10194,11 @@ } TARGET(STORE_ATTR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -8375,7 +10237,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { - goto pop_2_error; + JUMP_TO_LABEL(pop_2_error); } } stack_pointer += -2; @@ -8384,8 +10246,14 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8398,11 +10266,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } } // _GUARD_DORV_NO_DICT @@ -8413,7 +10289,11 @@ if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } } // _STORE_ATTR_INSTANCE_VALUE @@ -8443,8 +10323,14 @@ } TARGET(STORE_ATTR_SLOT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_SLOT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8457,14 +10343,22 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -8481,8 +10375,14 @@ } TARGET(STORE_ATTR_WITH_HINT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); @@ -8495,7 +10395,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } // _STORE_ATTR_WITH_HINT { @@ -8504,12 +10408,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, STORE_ATTR); - DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } #ifdef Py_GIL_DISABLED if (dict != _PyObject_GetManagedDict(owner_o)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } #endif assert(PyDict_CheckExact((PyObject *)dict)); @@ -8517,17 +10433,29 @@ if (hint >= (size_t)dict->ma_keys->dk_nentries || !DK_IS_UNICODE(dict->ma_keys)) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); - DEOPT_IF(true, STORE_ATTR); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); @@ -8548,6 +10476,11 @@ } TARGET(STORE_DEREF) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_DEREF; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -8563,6 +10496,11 @@ } TARGET(STORE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -8579,6 +10517,11 @@ } TARGET(STORE_FAST_LOAD_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -8598,6 +10541,11 @@ } TARGET(STORE_FAST_STORE_FAST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_FAST_STORE_FAST; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -8625,6 +10573,11 @@ } TARGET(STORE_GLOBAL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_GLOBAL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -8636,7 +10589,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8644,6 +10597,11 @@ } TARGET(STORE_NAME) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_NAME; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -8658,7 +10616,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(v); - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8672,7 +10630,7 @@ } PyStackRef_CLOSE(v); if (err) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8680,6 +10638,11 @@ } TARGET(STORE_SLICE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SLICE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -8721,7 +10684,7 @@ PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); if (err) { - goto pop_4_error; + JUMP_TO_LABEL(pop_4_error); } } stack_pointer += -4; @@ -8730,6 +10693,11 @@ } TARGET(STORE_SUBSCR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -8768,7 +10736,7 @@ PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); if (err) { - goto pop_3_error; + JUMP_TO_LABEL(pop_3_error); } } stack_pointer += -3; @@ -8777,6 +10745,13 @@ } TARGET(STORE_SUBSCR_DICT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_DICT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); @@ -8789,7 +10764,11 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -8802,12 +10781,19 @@ PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - goto error; + JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); @@ -8821,16 +10807,36 @@ value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + if (!PyLong_CheckExact(sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (!PyList_CheckExact(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR); + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } // Ensure index < len(list) if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); - DEOPT_IF(true, STORE_SUBSCR); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -8848,6 +10854,11 @@ } TARGET(SWAP) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = SWAP; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -8863,6 +10874,11 @@ } TARGET(TO_BOOL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -8896,7 +10912,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (err < 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = err ? PyStackRef_True : PyStackRef_False; } @@ -8905,8 +10921,14 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; + frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); @@ -8920,7 +10942,11 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } } // _REPLACE_WITH_TRUE { @@ -8933,6 +10959,13 @@ } TARGET(TO_BOOL_BOOL) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_BOOL; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); @@ -8941,12 +10974,23 @@ /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; - DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL); + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); DISPATCH(); } TARGET(TO_BOOL_INT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_INT; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); @@ -8957,7 +11001,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { assert(_Py_IsImmortal(value_o)); @@ -8972,6 +11020,13 @@ } TARGET(TO_BOOL_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); @@ -8982,7 +11037,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL); + if (!PyList_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; PyStackRef_CLOSE(value); @@ -8991,6 +11050,13 @@ } TARGET(TO_BOOL_NONE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_NONE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); @@ -9001,7 +11067,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL); + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; @@ -9009,6 +11079,13 @@ } TARGET(TO_BOOL_STR) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = TO_BOOL_STR; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); @@ -9019,7 +11096,11 @@ /* Skip 2 cache entries */ value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } STAT_INC(TO_BOOL, hit); if (value_o == &_Py_STR(empty)) { assert(_Py_IsImmortal(value_o)); @@ -9035,6 +11116,11 @@ } TARGET(UNARY_INVERT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_INVERT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -9046,7 +11132,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9054,6 +11140,11 @@ } TARGET(UNARY_NEGATIVE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NEGATIVE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -9065,7 +11156,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(value); if (res_o == NULL) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-1] = res; @@ -9073,6 +11164,11 @@ } TARGET(UNARY_NOT) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNARY_NOT; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -9087,6 +11183,11 @@ } TARGET(UNPACK_EX) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_EX; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -9100,7 +11201,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -9108,6 +11209,11 @@ } TARGET(UNPACK_SEQUENCE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -9144,7 +11250,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(seq); if (res == 0) { - goto pop_1_error; + JUMP_TO_LABEL(pop_1_error); } } stack_pointer += -1 + oparg; @@ -9153,6 +11259,13 @@ } TARGET(UNPACK_SEQUENCE_LIST) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); @@ -9163,11 +11276,23 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); + if (!PyList_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); - DEOPT_IF(true, UNPACK_SEQUENCE); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); @@ -9182,6 +11307,13 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); @@ -9192,8 +11324,16 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { @@ -9206,6 +11346,13 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); @@ -9217,8 +11364,16 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE); + if (!PyTuple_CheckExact(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); @@ -9231,6 +11386,11 @@ } TARGET(WITH_EXCEPT_START) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = WITH_EXCEPT_START; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -9275,7 +11435,7 @@ (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - goto error; + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -9285,6 +11445,11 @@ } TARGET(YIELD_VALUE) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode; + #endif + opcode = YIELD_VALUE; + (void)(opcode); frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -9331,6 +11496,7 @@ } /* END INSTRUCTIONS */ +#ifndef Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -9340,44 +11506,46 @@ next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); + } /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); +#endif /* Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ - pop_4_error: + LABEL(pop_4_error) { STACK_SHRINK(4); - goto error; + JUMP_TO_LABEL(error); } - pop_3_error: + LABEL(pop_3_error) { STACK_SHRINK(3); - goto error; + JUMP_TO_LABEL(error); } - pop_2_error: + LABEL(pop_2_error) { STACK_SHRINK(2); - goto error; + JUMP_TO_LABEL(error); } - pop_1_error: + LABEL(pop_1_error) { STACK_SHRINK(1); - goto error; + JUMP_TO_LABEL(error); } - error: + LABEL(error) { /* Double-check exception status. */ #ifdef NDEBUG @@ -9407,10 +11575,10 @@ _PyEval_MonitorRaise(tstate, frame, next_instr-1); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } - exception_unwind: + LABEL(exception_unwind) { /* STACK SPILLED */ /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -9427,7 +11595,7 @@ PyStackRef_XCLOSE(ref); } monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; @@ -9440,7 +11608,7 @@ int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); if (lasti == NULL) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); } @@ -9453,7 +11621,7 @@ next_instr = _PyFrame_GetBytecode(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { - goto exception_unwind; + JUMP_TO_LABEL(exception_unwind); } /* Resume normal execution */ #ifdef LLTRACE @@ -9462,10 +11630,13 @@ } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + int opcode; + #endif DISPATCH(); } - exit_unwind: + LABEL(exit_unwind) { /* STACK SPILLED */ assert(_PyErr_Occurred(tstate)); @@ -9484,15 +11655,15 @@ } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; + JUMP_TO_LABEL(error); } - start_frame: + LABEL(start_frame) { /* STACK SPILLED */ int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; #ifdef LLTRACE @@ -9500,7 +11671,7 @@ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); frame->lltrace = lltrace; if (lltrace < 0) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } } #endif @@ -9512,6 +11683,9 @@ assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #ifdef Py_TAIL_CALL_INTERP + int opcode; + #endif DISPATCH(); } |