summaryrefslogtreecommitdiffstats
path: root/Python/generated_cases.c.h
diff options
context:
space:
mode:
authorKen Jin <kenjin@python.org>2025-02-06 15:21:57 (GMT)
committerGitHub <noreply@github.com>2025-02-06 15:21:57 (GMT)
commitcb640b659e14cb0a05767054f95a9d25787b472d (patch)
treeab534fdb77f1dfb58fd704a53dc472099dc4dd8f /Python/generated_cases.c.h
parent555dc50c811e3e9ebdc30a1d511cf48a32666d6f (diff)
downloadcpython-cb640b659e14cb0a05767054f95a9d25787b472d.zip
cpython-cb640b659e14cb0a05767054f95a9d25787b472d.tar.gz
cpython-cb640b659e14cb0a05767054f95a9d25787b472d.tar.bz2
gh-128563: A new tail-calling interpreter (GH-128718)
Co-authored-by: Garrett Gu <garrettgu777@gmail.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Diffstat (limited to 'Python/generated_cases.c.h')
-rw-r--r--Python/generated_cases.c.h3232
1 files changed, 2703 insertions, 529 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 5820147..7927fa7 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -8,15 +8,22 @@
#endif
#define TIER_ONE 1
+#ifndef Py_TAIL_CALL_INTERP
#if !USE_COMPUTED_GOTOS
dispatch_opcode:
switch (opcode)
#endif
{
+#endif /* Py_TAIL_CALL_INTERP */
/* BEGIN INSTRUCTIONS */
TARGET(BINARY_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP);
@@ -58,7 +65,7 @@
PyStackRef_CLOSE(lhs);
PyStackRef_CLOSE(rhs);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -69,6 +76,13 @@
}
TARGET(BINARY_OP_ADD_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
@@ -82,8 +96,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_FLOAT
@@ -98,7 +120,7 @@
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -109,6 +131,13 @@
}
TARGET(BINARY_OP_ADD_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
@@ -122,8 +151,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_INT
@@ -137,7 +174,7 @@
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -148,6 +185,13 @@
}
TARGET(BINARY_OP_ADD_UNICODE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_ADD_UNICODE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
@@ -161,8 +205,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_ADD_UNICODE
@@ -176,7 +228,7 @@
PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -187,8 +239,14 @@
}
TARGET(BINARY_OP_EXTEND) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_EXTEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_EXTEND);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
@@ -209,7 +267,11 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = d->guard(left_o, right_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- DEOPT_IF(!res, BINARY_OP);
+ if (!res) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip -4 cache entry */
// _BINARY_OP_EXTEND
@@ -234,6 +296,13 @@
}
TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_INPLACE_ADD_UNICODE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
@@ -246,8 +315,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), BINARY_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_INPLACE_ADD_UNICODE
@@ -264,7 +341,11 @@
next_oparg = CURRENT_OPERAND0();
#endif
_PyStackRef *target_local = &GETLOCAL(next_oparg);
- DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o, BINARY_OP);
+ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
@@ -284,7 +365,7 @@
*target_local = PyStackRef_FromPyObjectSteal(temp);
PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (PyStackRef_IsNull(*target_local)) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
#if TIER_ONE
// The STORE_FAST is already done. This is done here in tier one,
@@ -299,6 +380,13 @@
}
TARGET(BINARY_OP_MULTIPLY_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_MULTIPLY_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
@@ -312,8 +400,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_FLOAT
@@ -328,7 +424,7 @@
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -339,6 +435,13 @@
}
TARGET(BINARY_OP_MULTIPLY_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_MULTIPLY_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
@@ -352,8 +455,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_INT
@@ -367,7 +478,7 @@
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -378,6 +489,13 @@
}
TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_SUBTRACT_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
@@ -391,8 +509,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), BINARY_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_FLOAT
@@ -407,7 +533,7 @@
((PyFloatObject *)right_o)->ob_fval;
PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -418,6 +544,13 @@
}
TARGET(BINARY_OP_SUBTRACT_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_OP_SUBTRACT_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
@@ -431,8 +564,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), BINARY_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), BINARY_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_INT
@@ -446,7 +587,7 @@
PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -457,6 +598,11 @@
}
TARGET(BINARY_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BINARY_SLICE);
@@ -502,7 +648,7 @@
PyStackRef_CLOSE(container);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -513,6 +659,11 @@
}
TARGET(BINARY_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR);
@@ -551,7 +702,7 @@
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -562,6 +713,13 @@
}
TARGET(BINARY_SUBSCR_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
@@ -574,7 +732,11 @@
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
- DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -588,7 +750,7 @@
PyStackRef_CLOSE(dict_st);
PyStackRef_CLOSE(sub_st);
if (rc <= 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
// not found or error
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -599,6 +761,13 @@
}
TARGET(BINARY_SUBSCR_GETITEM) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_GETITEM;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
@@ -610,22 +779,42 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
}
// _BINARY_SUBSCR_CHECK_FUNC
{
container = stack_pointer[-2];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
- DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
+ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem);
- DEOPT_IF(getitem_o == NULL, BINARY_SUBSCR);
+ if (getitem_o == NULL) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
assert(PyFunction_Check(getitem_o));
uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version);
- DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version, BINARY_SUBSCR);
+ if (((PyFunctionObject *)getitem_o)->func_version != cached_version) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o);
assert(code->co_argcount == 2);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
getitem = PyStackRef_FromPyObjectNew(getitem_o);
STAT_INC(BINARY_SUBSCR, hit);
}
@@ -658,6 +847,13 @@
}
TARGET(BINARY_SUBSCR_LIST_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_LIST_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
@@ -670,19 +866,39 @@
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyList_CheckExact(list)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Deopt unless 0 <= sub < PyList_Size(list)
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
- DEOPT_IF(res_o == NULL, BINARY_SUBSCR);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
#else
- DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
+ if (index >= PyList_GET_SIZE(list)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
@@ -702,6 +918,13 @@
}
TARGET(BINARY_SUBSCR_STR_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_STR_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
@@ -714,14 +937,34 @@
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyUnicode_CheckExact(str)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
+ if (PyUnicode_GET_LENGTH(str) <= index) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
- DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
+ if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
@@ -738,6 +981,13 @@
}
TARGET(BINARY_SUBSCR_TUPLE_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BINARY_SUBSCR_TUPLE_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
@@ -750,12 +1000,28 @@
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
- DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
- DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
+ if (!PyTuple_CheckExact(tuple)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
// Deopt unless 0 <= sub < PyTuple_Size(list)
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
+ if (index >= PyTuple_GET_SIZE(tuple)) {
+ UPDATE_MISS_STATS(BINARY_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
+ JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ }
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
@@ -774,6 +1040,11 @@
}
TARGET(BUILD_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_LIST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_LIST);
@@ -782,7 +1053,7 @@
values = &stack_pointer[-oparg];
PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg);
if (list_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
list = PyStackRef_FromPyObjectSteal(list_o);
stack_pointer[-oparg] = list;
@@ -792,6 +1063,11 @@
}
TARGET(BUILD_MAP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_MAP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_MAP);
@@ -805,7 +1081,7 @@
}
stack_pointer += -oparg*2;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *map_o = _PyDict_FromItems(
@@ -820,7 +1096,7 @@
if (map_o == NULL) {
stack_pointer += -oparg*2;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
map = PyStackRef_FromPyObjectSteal(map_o);
stack_pointer[-oparg*2] = map;
@@ -830,6 +1106,11 @@
}
TARGET(BUILD_SET) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_SET;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SET);
@@ -845,7 +1126,7 @@
}
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
int err = 0;
for (int i = 0; i < oparg; i++) {
@@ -864,7 +1145,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(set_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
set = PyStackRef_FromPyObjectSteal(set_o);
stack_pointer[-oparg] = set;
@@ -874,6 +1155,11 @@
}
TARGET(BUILD_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
@@ -890,7 +1176,7 @@
if (slice_o == NULL) {
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-oparg] = slice;
@@ -900,6 +1186,11 @@
}
TARGET(BUILD_STRING) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_STRING;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_STRING);
@@ -913,7 +1204,7 @@
}
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
@@ -923,7 +1214,7 @@
if (str_o == NULL) {
stack_pointer += -oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
str = PyStackRef_FromPyObjectSteal(str_o);
stack_pointer[-oparg] = str;
@@ -933,6 +1224,11 @@
}
TARGET(BUILD_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = BUILD_TUPLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_TUPLE);
@@ -941,7 +1237,7 @@
values = &stack_pointer[-oparg];
PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg);
if (tup_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
tup = PyStackRef_FromPyObjectSteal(tup_o);
stack_pointer[-oparg] = tup;
@@ -951,6 +1247,11 @@
}
TARGET(CACHE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CACHE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CACHE);
@@ -960,13 +1261,17 @@
}
TARGET(CALL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL);
PREDICTED_CALL:;
_Py_CODEUNIT* const this_instr = next_instr - 4;
(void)this_instr;
- opcode = CALL;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
@@ -1041,7 +1346,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
frame->return_offset = 4 ;
DISPATCH_INLINED(new_frame);
@@ -1056,7 +1361,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
@@ -1097,7 +1402,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -1113,7 +1418,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -1126,8 +1431,14 @@
}
TARGET(CALL_ALLOC_AND_ENTER_INIT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_ALLOC_AND_ENTER_INIT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
@@ -1141,7 +1452,11 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_AND_ALLOCATE_OBJECT
{
@@ -1152,23 +1467,39 @@
self = &stack_pointer[-1 - oparg];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
- DEOPT_IF(!PyType_Check(callable_o), CALL);
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyType_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *tp = (PyTypeObject *)callable_o;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
assert(tp->tp_new == PyBaseObject_Type.tp_new);
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
assert(tp->tp_alloc == PyType_GenericAlloc);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init);
PyCodeObject *code = (PyCodeObject *)init_func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *self_o = PyType_GenericAlloc(tp, 0);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (self_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
self[0] = PyStackRef_FromPyObjectSteal(self_o);
_PyStackRef temp = callable[0];
@@ -1198,7 +1529,7 @@
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
_PyEval_FrameClearAndPop(tstate, shim);
- goto error;
+ JUMP_TO_LABEL(error);
}
init_frame = temp;
frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
@@ -1227,8 +1558,14 @@
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BOUND_METHOD_EXACT_ARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
@@ -1240,14 +1577,26 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
{
null = &stack_pointer[-1 - oparg];
callable = &stack_pointer[-2 - oparg];
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type, CALL);
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable[0])) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
@@ -1267,9 +1616,17 @@
{
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_EXACT_ARGS
{
@@ -1277,15 +1634,27 @@
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL);
+ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
- DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (tstate->py_recursion_remaining <= 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_PY_EXACT_ARGS
{
@@ -1329,8 +1698,14 @@
}
TARGET(CALL_BOUND_METHOD_GENERAL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BOUND_METHOD_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
@@ -1342,7 +1717,11 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_METHOD_VERSION
{
@@ -1350,11 +1729,27 @@
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL);
+ if (Py_TYPE(callable_o) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
- DEOPT_IF(!PyFunction_Check(func), CALL);
- DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL);
+ if (!PyFunction_Check(func)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (((PyFunctionObject *)func)->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _EXPAND_METHOD
{
@@ -1394,7 +1789,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
@@ -1426,6 +1821,13 @@
}
TARGET(CALL_BUILTIN_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_CLASS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
@@ -1442,7 +1844,11 @@
self_or_null = &stack_pointer[-1 - oparg];
callable = &stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyType_Check(callable_o), CALL);
+ if (!PyType_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *tp = (PyTypeObject *)callable_o;
int total_args = oparg;
_PyStackRef *arguments = args;
@@ -1450,7 +1856,11 @@
arguments--;
total_args++;
}
- DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
+ if (tp->tp_vectorcall == NULL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
@@ -1461,7 +1871,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
@@ -1475,7 +1885,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -1491,7 +1901,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -1504,6 +1914,13 @@
}
TARGET(CALL_BUILTIN_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_FAST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
@@ -1527,8 +1944,16 @@
arguments--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL, CALL);
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
@@ -1541,7 +1966,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = ((PyCFunctionFast)(void(*)(void))cfunc)(
@@ -1559,7 +1984,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -1575,7 +2000,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -1588,6 +2013,13 @@
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
@@ -1611,8 +2043,16 @@
arguments--;
total_args++;
}
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS), CALL);
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
/* res = func(self, arguments, nargs, kwnames) */
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -1629,7 +2069,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
@@ -1644,7 +2084,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -1660,7 +2100,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -1673,6 +2113,13 @@
}
TARGET(CALL_BUILTIN_O) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_BUILTIN_O;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
@@ -1695,11 +2142,27 @@
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
- DEOPT_IF(!PyCFunction_CheckExact(callable_o), CALL);
- DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!PyCFunction_CheckExact(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (PyCFunction_GET_FLAGS(callable_o) != METH_O) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
_PyStackRef arg = args[0];
@@ -1718,7 +2181,7 @@
PyStackRef_CLOSE(callable[0]);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -1734,7 +2197,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -1747,11 +2210,16 @@
}
TARGET(CALL_FUNCTION_EX) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_FUNCTION_EX;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
- opcode = CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef kwargs_in;
@@ -1777,13 +2245,13 @@
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
kwargs_out = kwargs_in;
stack_pointer += -2;
@@ -1824,7 +2292,7 @@
frame, this_instr, func, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
result_o = PyObject_Call(func, callargs, kwargs);
@@ -1873,7 +2341,7 @@
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 1 == 1);
frame->return_offset = 1;
@@ -1905,7 +2373,7 @@
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
@@ -1921,7 +2389,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -1934,6 +2402,11 @@
}
TARGET(CALL_INTRINSIC_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_INTRINSIC_1;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_1);
@@ -1946,7 +2419,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
@@ -1954,6 +2427,11 @@
}
TARGET(CALL_INTRINSIC_2) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_INTRINSIC_2;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_2);
@@ -1971,7 +2449,7 @@
PyStackRef_CLOSE(value2_st);
PyStackRef_CLOSE(value1_st);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -1981,6 +2459,13 @@
}
TARGET(CALL_ISINSTANCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_ISINSTANCE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
@@ -2002,9 +2487,17 @@
arguments--;
total_args++;
}
- DEOPT_IF(total_args != 2, CALL);
+ if (total_args != 2) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.isinstance, CALL);
+ if (callable_o != interp->callable_cache.isinstance) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyStackRef cls_stackref = arguments[1];
_PyStackRef inst_stackref = arguments[0];
@@ -2012,7 +2505,7 @@
int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (retval < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = retval ? PyStackRef_True : PyStackRef_False;
assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -2028,13 +2521,17 @@
}
TARGET(CALL_KW) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW);
PREDICTED_CALL_KW:;
_Py_CODEUNIT* const this_instr = next_instr - 4;
(void)this_instr;
- opcode = CALL_KW;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
@@ -2123,7 +2620,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
frame->return_offset = 4 ;
@@ -2140,7 +2637,7 @@
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer[-1] = kwnames;
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -2182,7 +2679,7 @@
if (res_o == NULL) {
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2193,8 +2690,14 @@
}
TARGET(CALL_KW_BOUND_METHOD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_BOUND_METHOD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_BOUND_METHOD);
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
@@ -2207,7 +2710,11 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL_KW);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CHECK_METHOD_VERSION_KW
{
@@ -2215,11 +2722,27 @@
callable = &stack_pointer[-3 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(Py_TYPE(callable_o) != &PyMethod_Type, CALL_KW);
+ if (Py_TYPE(callable_o) != &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
- DEOPT_IF(!PyFunction_Check(func), CALL_KW);
- DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL_KW);
- DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL_KW);
+ if (!PyFunction_Check(func)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (((PyFunctionObject *)func)->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (!PyStackRef_IsNull(null[0])) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _EXPAND_METHOD_KW
{
@@ -2269,7 +2792,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
@@ -2301,10 +2824,16 @@
}
TARGET(CALL_KW_NON_PY) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_NON_PY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_NON_PY);
- opcode = CALL_KW_NON_PY;
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
_PyStackRef *callable;
_PyStackRef kwnames;
@@ -2317,8 +2846,16 @@
{
callable = &stack_pointer[-3 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(PyFunction_Check(callable_o), CALL_KW);
- DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL_KW);
+ if (PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
+ if (Py_TYPE(callable_o) == &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CALL_KW_NON_PY
{
@@ -2346,7 +2883,7 @@
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
@@ -2371,7 +2908,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2387,7 +2924,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -2400,8 +2937,14 @@
}
TARGET(CALL_KW_PY) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_KW_PY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_KW_PY);
static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size");
@@ -2413,16 +2956,28 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL_KW);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _CHECK_FUNCTION_VERSION_KW
{
callable = &stack_pointer[-3 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL_KW);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL_KW);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL_KW);
+ assert(_PyOpcode_Deopt[opcode] == (CALL_KW));
+ JUMP_TO_PREDICTED(CALL_KW);
+ }
}
// _PY_FRAME_KW
{
@@ -2458,7 +3013,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
@@ -2490,6 +3045,13 @@
}
TARGET(CALL_LEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_LEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
@@ -2510,9 +3072,17 @@
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.len, CALL);
+ if (callable_o != interp->callable_cache.len) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyStackRef arg_stackref = args[0];
PyObject *arg = PyStackRef_AsPyObjectBorrow(arg_stackref);
@@ -2520,7 +3090,7 @@
Py_ssize_t len_i = PyObject_Length(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (len_i < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *res_o = PyLong_FromSsize_t(len_i);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -2543,6 +3113,13 @@
}
TARGET(CALL_LIST_APPEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_LIST_APPEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
@@ -2559,10 +3136,22 @@
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *self_o = PyStackRef_AsPyObjectBorrow(self);
PyInterpreterState *interp = tstate->interp;
- DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL);
+ if (callable_o != interp->callable_cache.list_append) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
assert(self_o != NULL);
- DEOPT_IF(!PyList_Check(self_o), CALL);
- DEOPT_IF(!LOCK_OBJECT(self_o), CALL);
+ if (!PyList_Check(self_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!LOCK_OBJECT(self_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg));
UNLOCK_OBJECT(self_o);
@@ -2577,7 +3166,7 @@
PyStackRef_CLOSE(callable);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
#if TIER_ONE
// Skip the following POP_TOP. This is done here in tier one, and
@@ -2589,6 +3178,13 @@
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_FAST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
@@ -2613,11 +3209,23 @@
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
/* Builtin METH_FASTCALL methods, without keywords */
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
+ if (meth->ml_flags != METH_FASTCALL) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
@@ -2629,7 +3237,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyCFunctionFast cfunc =
@@ -2646,7 +3254,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2662,7 +3270,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -2675,6 +3283,13 @@
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
@@ -2698,12 +3313,24 @@
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
+ if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
- DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
+ if (!Py_IS_TYPE(self, d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
@@ -2715,7 +3342,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyCFunctionFastWithKeywords cfunc =
@@ -2732,7 +3359,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2748,7 +3375,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -2761,6 +3388,13 @@
}
TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_NOARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
@@ -2783,16 +3417,36 @@
args--;
total_args++;
}
- DEOPT_IF(total_args != 1, CALL);
+ if (total_args != 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
_PyStackRef self_stackref = args[0];
PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
- DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
- DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
+ if (!Py_IS_TYPE(self, method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (meth->ml_flags != METH_NOARGS) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
@@ -2810,7 +3464,7 @@
PyStackRef_CLOSE(callable[0]);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2826,7 +3480,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -2839,6 +3493,13 @@
}
TARGET(CALL_METHOD_DESCRIPTOR_O) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_METHOD_DESCRIPTOR_O;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
@@ -2862,16 +3523,36 @@
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
- DEOPT_IF(total_args != 2, CALL);
- DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
+ if (total_args != 2) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyMethodDef *meth = method->d_method;
- DEOPT_IF(meth->ml_flags != METH_O, CALL);
+ if (meth->ml_flags != METH_O) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
// CPython promises to check all non-vectorcall function calls.
- DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
+ if (tstate->c_recursion_remaining <= 0) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
_PyStackRef arg_stackref = arguments[1];
_PyStackRef self_stackref = arguments[0];
- DEOPT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
- method->d_common.d_type), CALL);
+ if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
+ method->d_common.d_type)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
@@ -2890,7 +3571,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2906,7 +3587,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -2919,10 +3600,16 @@
}
TARGET(CALL_NON_PY_GENERAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_NON_PY_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_NON_PY_GENERAL);
- opcode = CALL_NON_PY_GENERAL;
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
_PyStackRef *callable;
_PyStackRef *self_or_null;
@@ -2934,8 +3621,16 @@
{
callable = &stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(PyFunction_Check(callable_o), CALL);
- DEOPT_IF(Py_TYPE(callable_o) == &PyMethod_Type, CALL);
+ if (PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (Py_TYPE(callable_o) == &PyMethod_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CALL_NON_PY_GENERAL
{
@@ -2961,7 +3656,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
@@ -2979,7 +3674,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -2995,7 +3690,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -3008,8 +3703,14 @@
}
TARGET(CALL_PY_EXACT_ARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_PY_EXACT_ARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
@@ -3020,16 +3721,28 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_VERSION
{
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_EXACT_ARGS
{
@@ -3038,15 +3751,27 @@
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0])), CALL);
+ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null[0]))) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_STACK_SPACE
{
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
PyCodeObject *code = (PyCodeObject *)func->func_code;
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
- DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (tstate->py_recursion_remaining <= 1) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _INIT_CALL_PY_EXACT_ARGS
{
@@ -3090,8 +3815,14 @@
}
TARGET(CALL_PY_GENERAL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_PY_GENERAL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
@@ -3102,16 +3833,28 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, CALL);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _CHECK_FUNCTION_VERSION
{
callable = &stack_pointer[-2 - oparg];
uint32_t func_version = read_u32(&this_instr[2].cache);
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
- DEOPT_IF(!PyFunction_Check(callable_o), CALL);
+ if (!PyFunction_Check(callable_o)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- DEOPT_IF(func->func_version != func_version, CALL);
+ if (func->func_version != func_version) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
}
// _PY_FRAME_GENERAL
{
@@ -3137,7 +3880,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
if (temp == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
new_frame = temp;
}
@@ -3169,6 +3912,13 @@
}
TARGET(CALL_STR_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_STR_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
@@ -3187,8 +3937,16 @@
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyUnicode_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyUnicode_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Str(arg_o);
@@ -3199,7 +3957,7 @@
PyStackRef_CLOSE(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -3215,7 +3973,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -3228,6 +3986,13 @@
}
TARGET(CALL_TUPLE_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_TUPLE_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
@@ -3246,8 +4011,16 @@
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyTuple_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PySequence_Tuple(arg_o);
@@ -3258,7 +4031,7 @@
PyStackRef_CLOSE(arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -3274,7 +4047,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -3287,6 +4060,13 @@
}
TARGET(CALL_TYPE_1) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CALL_TYPE_1;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
@@ -3303,8 +4083,16 @@
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg);
assert(oparg == 1);
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(callable_o != (PyObject *)&PyType_Type, CALL);
+ if (!PyStackRef_IsNull(null)) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
+ if (callable_o != (PyObject *)&PyType_Type) {
+ UPDATE_MISS_STATS(CALL);
+ assert(_PyOpcode_Deopt[opcode] == (CALL));
+ JUMP_TO_PREDICTED(CALL);
+ }
STAT_INC(CALL, hit);
res = PyStackRef_FromPyObjectSteal(Py_NewRef(Py_TYPE(arg_o)));
stack_pointer[-3] = res;
@@ -3317,6 +4105,11 @@
}
TARGET(CHECK_EG_MATCH) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CHECK_EG_MATCH;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EG_MATCH);
@@ -3334,7 +4127,7 @@
if (err < 0) {
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
PyObject *match_o = NULL;
PyObject *rest_o = NULL;
@@ -3345,11 +4138,11 @@
PyStackRef_CLOSE(exc_value_st);
PyStackRef_CLOSE(match_type_st);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
assert((match_o == NULL) == (rest_o == NULL));
if (match_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
if (!Py_IsNone(match_o)) {
stack_pointer += -2;
@@ -3368,6 +4161,11 @@
}
TARGET(CHECK_EXC_MATCH) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CHECK_EXC_MATCH;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EXC_MATCH);
@@ -3384,7 +4182,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
PyStackRef_CLOSE(right);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyErr_GivenExceptionMatches(left_o, right_o);
@@ -3396,8 +4194,14 @@
}
TARGET(CLEANUP_THROW) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CLEANUP_THROW;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CLEANUP_THROW);
_PyStackRef sub_iter_st;
@@ -3409,7 +4213,9 @@
last_sent_val_st = stack_pointer[-2];
sub_iter_st = stack_pointer[-3];
PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
+ #ifndef Py_TAIL_CALL_INTERP
assert(throwflag);
+ #endif
assert(exc_value && PyExceptionInstance_Check(exc_value));
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration);
@@ -3427,7 +4233,7 @@
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
stack_pointer[-3] = none;
stack_pointer[-2] = value;
@@ -3437,6 +4243,11 @@
}
TARGET(COMPARE_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP);
@@ -3475,7 +4286,7 @@
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
if (oparg & 16) {
stack_pointer += -2;
@@ -3485,7 +4296,7 @@
Py_DECREF(res_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_bool < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = res_bool ? PyStackRef_True : PyStackRef_False;
}
@@ -3502,6 +4313,13 @@
}
TARGET(COMPARE_OP_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_FLOAT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
@@ -3515,8 +4333,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyFloat_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyFloat_CheckExact(right_o), COMPARE_OP);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
@@ -3540,6 +4366,13 @@
}
TARGET(COMPARE_OP_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
@@ -3553,16 +4386,32 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyLong_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyLong_CheckExact(right_o), COMPARE_OP);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o), COMPARE_OP);
- DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o), COMPARE_OP);
+ if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!_PyLong_IsCompact((PyLongObject *)right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right_o) <= 1);
@@ -3582,6 +4431,13 @@
}
TARGET(COMPARE_OP_STR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COMPARE_OP_STR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
@@ -3595,8 +4451,16 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyUnicode_CheckExact(left_o), COMPARE_OP);
- DEOPT_IF(!PyUnicode_CheckExact(right_o), COMPARE_OP);
+ if (!PyUnicode_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
+ if (!PyUnicode_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(COMPARE_OP);
+ assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
+ JUMP_TO_PREDICTED(COMPARE_OP);
+ }
}
/* Skip 1 cache entry */
// _COMPARE_OP_STR
@@ -3621,6 +4485,11 @@
}
TARGET(CONTAINS_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP);
@@ -3658,7 +4527,7 @@
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
@@ -3669,6 +4538,13 @@
}
TARGET(CONTAINS_OP_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
@@ -3681,7 +4557,11 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!PyDict_CheckExact(right_o), CONTAINS_OP);
+ if (!PyDict_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(CONTAINS_OP);
+ assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
+ JUMP_TO_PREDICTED(CONTAINS_OP);
+ }
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyDict_Contains(right_o, left_o);
@@ -3689,7 +4569,7 @@
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
@@ -3699,6 +4579,13 @@
}
TARGET(CONTAINS_OP_SET) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONTAINS_OP_SET;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
@@ -3711,7 +4598,11 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)), CONTAINS_OP);
+ if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) {
+ UPDATE_MISS_STATS(CONTAINS_OP);
+ assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
+ JUMP_TO_PREDICTED(CONTAINS_OP);
+ }
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -3720,7 +4611,7 @@
PyStackRef_CLOSE(left);
PyStackRef_CLOSE(right);
if (res < 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[-2] = b;
@@ -3730,6 +4621,11 @@
}
TARGET(CONVERT_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = CONVERT_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CONVERT_VALUE);
@@ -3748,7 +4644,7 @@
PyStackRef_CLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
stack_pointer[0] = result;
@@ -3758,6 +4654,11 @@
}
TARGET(COPY) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COPY;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY);
@@ -3773,6 +4674,11 @@
}
TARGET(COPY_FREE_VARS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = COPY_FREE_VARS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY_FREE_VARS);
@@ -3791,6 +4697,11 @@
}
TARGET(DELETE_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_ATTR);
@@ -3802,7 +4713,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -3810,6 +4721,11 @@
}
TARGET(DELETE_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_DEREF);
@@ -3821,7 +4737,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(oldobj);
@@ -3830,6 +4746,11 @@
}
TARGET(DELETE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
@@ -3841,7 +4762,7 @@
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = PyStackRef_NULL;
@@ -3852,6 +4773,11 @@
}
TARGET(DELETE_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_GLOBAL);
@@ -3861,19 +4787,24 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
// Can't use ERROR_IF here.
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (err == 0) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(DELETE_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_NAME);
@@ -3885,7 +4816,7 @@
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
err = PyObject_DelItem(ns, name);
@@ -3897,12 +4828,17 @@
NAME_ERROR_MSG,
name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(DELETE_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DELETE_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_SUBSCR);
@@ -3918,7 +4854,7 @@
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
@@ -3926,6 +4862,11 @@
}
TARGET(DICT_MERGE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DICT_MERGE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_MERGE);
@@ -3946,7 +4887,7 @@
_PyEval_FormatKwargsError(tstate, callable_o, update_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(update);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
@@ -3955,6 +4896,11 @@
}
TARGET(DICT_UPDATE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = DICT_UPDATE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_UPDATE);
@@ -3979,7 +4925,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
}
PyStackRef_CLOSE(update);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
PyStackRef_CLOSE(update);
stack_pointer += -1;
@@ -3988,8 +4934,14 @@
}
TARGET(END_ASYNC_FOR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_ASYNC_FOR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_ASYNC_FOR);
_PyStackRef awaitable_st;
@@ -4012,7 +4964,7 @@
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
@@ -4020,6 +4972,11 @@
}
TARGET(END_FOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_FOR;
+ (void)(opcode);
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
_PyStackRef value;
@@ -4038,6 +4995,11 @@
}
TARGET(END_SEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = END_SEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_SEND);
@@ -4056,11 +5018,16 @@
}
TARGET(ENTER_EXECUTOR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = ENTER_EXECUTOR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(ENTER_EXECUTOR);
- opcode = ENTER_EXECUTOR;
#ifdef _Py_TIER2
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
@@ -4090,6 +5057,11 @@
}
TARGET(EXIT_INIT_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = EXIT_INIT_CHECK;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXIT_INIT_CHECK);
@@ -4102,7 +5074,7 @@
"__init__() should return None, not '%.200s'",
Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -4110,10 +5082,14 @@
}
TARGET(EXTENDED_ARG) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = EXTENDED_ARG;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXTENDED_ARG);
- opcode = EXTENDED_ARG;
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
@@ -4122,6 +5098,11 @@
}
TARGET(FORMAT_SIMPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FORMAT_SIMPLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_SIMPLE);
@@ -4141,7 +5122,7 @@
PyStackRef_CLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -4157,6 +5138,11 @@
}
TARGET(FORMAT_WITH_SPEC) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FORMAT_WITH_SPEC;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_WITH_SPEC);
@@ -4171,7 +5157,7 @@
PyStackRef_CLOSE(value);
PyStackRef_CLOSE(fmt_spec);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -4181,6 +5167,11 @@
}
TARGET(FOR_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER);
@@ -4219,7 +5210,7 @@
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
@@ -4243,6 +5234,13 @@
}
TARGET(FOR_ITER_GEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_GEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
@@ -4253,14 +5251,26 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _FOR_ITER_GEN_FRAME
{
iter = stack_pointer[-1];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
- DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
- DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
+ if (Py_TYPE(gen) != &PyGen_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
+ if (gen->gi_frame_state >= FRAME_EXECUTING) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;
_PyFrame_StackPush(gen_frame, PyStackRef_None);
@@ -4291,6 +5301,13 @@
}
TARGET(FOR_ITER_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
@@ -4301,7 +5318,11 @@
// _ITER_CHECK_LIST
{
iter = stack_pointer[-1];
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type, FOR_ITER);
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_LIST
{
@@ -4342,6 +5363,13 @@
}
TARGET(FOR_ITER_RANGE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_RANGE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
@@ -4353,7 +5381,11 @@
{
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
- DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
+ if (Py_TYPE(r) != &PyRangeIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_RANGE
{
@@ -4376,7 +5408,7 @@
r->len--;
PyObject *res = PyLong_FromLong(value);
if (res == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next = PyStackRef_FromPyObjectSteal(res);
}
@@ -4387,6 +5419,13 @@
}
TARGET(FOR_ITER_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = FOR_ITER_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
@@ -4397,7 +5436,11 @@
// _ITER_CHECK_TUPLE
{
iter = stack_pointer[-1];
- DEOPT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type, FOR_ITER);
+ if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) {
+ UPDATE_MISS_STATS(FOR_ITER);
+ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
+ JUMP_TO_PREDICTED(FOR_ITER);
+ }
}
// _ITER_JUMP_TUPLE
{
@@ -4435,6 +5478,11 @@
}
TARGET(GET_AITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_AITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AITER);
@@ -4456,14 +5504,14 @@
type->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(obj);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
iter_o = (*getter)(obj_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(obj);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (Py_TYPE(iter_o)->tp_as_async == NULL ||
Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) {
@@ -4476,7 +5524,7 @@
Py_TYPE(iter_o)->tp_name);
Py_DECREF(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
@@ -4484,6 +5532,11 @@
}
TARGET(GET_ANEXT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_ANEXT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ANEXT);
@@ -4494,7 +5547,7 @@
PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (awaitable_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
awaitable = PyStackRef_FromPyObjectSteal(awaitable_o);
stack_pointer[0] = awaitable;
@@ -4504,6 +5557,11 @@
}
TARGET(GET_AWAITABLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_AWAITABLE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AWAITABLE);
@@ -4515,7 +5573,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
@@ -4523,6 +5581,11 @@
}
TARGET(GET_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ITER);
@@ -4535,7 +5598,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (iter_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
stack_pointer[-1] = iter;
@@ -4543,6 +5606,11 @@
}
TARGET(GET_LEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_LEN;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_LEN);
@@ -4554,11 +5622,11 @@
Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (len_i < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
PyObject *len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
len = PyStackRef_FromPyObjectSteal(len_o);
stack_pointer[0] = len;
@@ -4568,6 +5636,11 @@
}
TARGET(GET_YIELD_FROM_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = GET_YIELD_FROM_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
@@ -4586,7 +5659,7 @@
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = iterable;
}
@@ -4600,7 +5673,7 @@
PyObject *iter_o = PyObject_GetIter(iterable_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (iter_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
PyStackRef_CLOSE(iterable);
@@ -4611,6 +5684,11 @@
}
TARGET(IMPORT_FROM) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IMPORT_FROM;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_FROM);
@@ -4622,7 +5700,7 @@
PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
@@ -4632,6 +5710,11 @@
}
TARGET(IMPORT_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IMPORT_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_NAME);
@@ -4649,7 +5732,7 @@
PyStackRef_CLOSE(level);
PyStackRef_CLOSE(fromlist);
if (res_o == NULL) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -4659,11 +5742,16 @@
}
TARGET(INSTRUMENTED_CALL) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL);
- opcode = INSTRUMENTED_CALL;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
@@ -4716,7 +5804,7 @@
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _DO_CALL
@@ -4750,7 +5838,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
frame->return_offset = 4 ;
DISPATCH_INLINED(new_frame);
@@ -4765,7 +5853,7 @@
}
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = PyObject_Vectorcall(
@@ -4806,7 +5894,7 @@
if (res_o == NULL) {
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -4822,7 +5910,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@@ -4835,11 +5923,16 @@
}
TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL_FUNCTION_EX;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
- opcode = INSTRUMENTED_CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef kwargs_in;
@@ -4865,13 +5958,13 @@
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
kwargs_out = kwargs_in;
stack_pointer += -2;
@@ -4912,7 +6005,7 @@
frame, this_instr, func, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
result_o = PyObject_Call(func, callargs, kwargs);
@@ -4961,7 +6054,7 @@
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 1 == 1);
frame->return_offset = 1;
@@ -4993,7 +6086,7 @@
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
@@ -5009,7 +6102,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -5022,11 +6115,16 @@
}
TARGET(INSTRUMENTED_CALL_KW) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_CALL_KW;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
- opcode = INSTRUMENTED_CALL_KW;
_PyStackRef *callable;
_PyStackRef *self_or_null;
_PyStackRef *args;
@@ -5063,7 +6161,7 @@
frame, this_instr, function, arg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _MAYBE_EXPAND_METHOD_KW
@@ -5125,7 +6223,7 @@
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW);
frame->return_offset = 4 ;
@@ -5142,7 +6240,7 @@
PyStackRef_CLOSE(kwnames);
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer[-1] = kwnames;
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -5184,7 +6282,7 @@
if (res_o == NULL) {
stack_pointer += -3 - oparg;
assert(WITHIN_STACK_BOUNDS());
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -5195,6 +6293,11 @@
}
TARGET(INSTRUMENTED_END_FOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_END_FOR;
+ (void)(opcode);
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
next_instr += 1;
@@ -5210,7 +6313,7 @@
int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
PyStackRef_CLOSE(value);
@@ -5220,8 +6323,14 @@
}
TARGET(INSTRUMENTED_END_SEND) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_END_SEND;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
_PyStackRef receiver;
@@ -5235,7 +6344,7 @@
int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
val = value;
@@ -5249,8 +6358,14 @@
}
TARGET(INSTRUMENTED_FOR_ITER) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_FOR_ITER;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
/* Skip 1 cache entry */
@@ -5269,7 +6384,7 @@
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
@@ -5286,17 +6401,22 @@
}
TARGET(INSTRUMENTED_INSTRUCTION) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_INSTRUCTION;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION);
- opcode = INSTRUMENTED_INSTRUCTION;
_PyFrame_SetStackPointer(frame, stack_pointer);
int next_opcode = _Py_call_instrumentation_instruction(
tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_opcode < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) {
@@ -5308,8 +6428,14 @@
}
TARGET(INSTRUMENTED_JUMP_BACKWARD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_JUMP_BACKWARD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
/* Skip 1 cache entry */
@@ -5322,7 +6448,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -5334,8 +6460,14 @@
}
TARGET(INSTRUMENTED_JUMP_FORWARD) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_JUMP_FORWARD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD);
INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
@@ -5343,12 +6475,17 @@
}
TARGET(INSTRUMENTED_LINE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_LINE;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_LINE);
- opcode = INSTRUMENTED_LINE;
int original_opcode = 0;
if (tstate->tracing) {
PyCodeObject *code = _PyFrame_GetCode(frame);
@@ -5364,7 +6501,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
if (original_opcode < 0) {
next_instr = this_instr+1;
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = frame->instr_ptr;
if (next_instr != this_instr) {
@@ -5382,11 +6519,16 @@
}
TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_LOAD_SUPER_ATTR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR);
- opcode = INSTRUMENTED_LOAD_SUPER_ATTR;
_PyStackRef global_super_st;
_PyStackRef class_st;
_PyStackRef self_st;
@@ -5412,7 +6554,7 @@
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
// we make no attempt to optimize here; specializations should
@@ -5447,7 +6589,7 @@
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
stack_pointer += -3;
@@ -5457,7 +6599,7 @@
Py_DECREF(super);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
@@ -5473,9 +6615,15 @@
}
TARGET(INSTRUMENTED_NOT_TAKEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_NOT_TAKEN;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN);
(void)this_instr; // INSTRUMENTED_JUMP requires this_instr
@@ -5484,9 +6632,15 @@
}
TARGET(INSTRUMENTED_POP_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_ITER;
+ (void)(opcode);
_Py_CODEUNIT* const prev_instr = frame->instr_ptr;
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_POP_ITER);
_PyStackRef iter;
@@ -5501,8 +6655,14 @@
}
TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_FALSE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
/* Skip 1 cache entry */
@@ -5517,8 +6677,14 @@
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
/* Skip 1 cache entry */
@@ -5537,8 +6703,14 @@
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
/* Skip 1 cache entry */
@@ -5555,8 +6727,14 @@
}
TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_POP_JUMP_IF_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
/* Skip 1 cache entry */
@@ -5571,8 +6749,14 @@
}
TARGET(INSTRUMENTED_RESUME) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_RESUME;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
// _LOAD_BYTECODE
@@ -5585,7 +6769,7 @@
_PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (bytecode == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
@@ -5609,7 +6793,7 @@
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
DISPATCH();
@@ -5626,7 +6810,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -5638,7 +6822,7 @@
tstate, oparg > 0, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (frame->instr_ptr != this_instr) {
/* Instrumentation has jumped */
@@ -5649,8 +6833,14 @@
}
TARGET(INSTRUMENTED_RETURN_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_RETURN_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
_PyStackRef val;
@@ -5665,7 +6855,7 @@
frame, this_instr, PyStackRef_AsPyObjectBorrow(val));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _RETURN_VALUE
@@ -5694,8 +6884,14 @@
}
TARGET(INSTRUMENTED_YIELD_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INSTRUMENTED_YIELD_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
_PyStackRef val;
@@ -5710,7 +6906,7 @@
frame, this_instr, PyStackRef_AsPyObjectBorrow(val));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (frame->instr_ptr != this_instr) {
next_instr = frame->instr_ptr;
@@ -5761,6 +6957,11 @@
}
TARGET(INTERPRETER_EXIT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = INTERPRETER_EXIT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INTERPRETER_EXIT);
@@ -5780,6 +6981,11 @@
}
TARGET(IS_OP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = IS_OP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IS_OP);
@@ -5799,6 +7005,11 @@
}
TARGET(JUMP_BACKWARD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
@@ -5826,7 +7037,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -5844,8 +7055,14 @@
}
TARGET(JUMP_BACKWARD_JIT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_JIT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD_JIT);
static_assert(1 == 1, "incorrect cache size");
@@ -5859,7 +7076,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -5891,7 +7108,7 @@
if (optimized <= 0) {
this_instr[1].counter = restart_backoff_counter(counter);
if (optimized < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
@@ -5912,6 +7129,11 @@
}
TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_NO_INTERRUPT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT);
@@ -5926,6 +7148,11 @@
}
TARGET(JUMP_BACKWARD_NO_JIT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_BACKWARD_NO_JIT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT);
@@ -5940,7 +7167,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -5958,6 +7185,11 @@
}
TARGET(JUMP_FORWARD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = JUMP_FORWARD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_FORWARD);
@@ -5966,6 +7198,11 @@
}
TARGET(LIST_APPEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LIST_APPEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_APPEND);
@@ -5976,7 +7213,7 @@
int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
PyStackRef_AsPyObjectSteal(v));
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -5984,6 +7221,11 @@
}
TARGET(LIST_EXTEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LIST_EXTEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_EXTEND);
@@ -6011,7 +7253,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
}
PyStackRef_CLOSE(iterable_st);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
assert(Py_IsNone(none_val));
PyStackRef_CLOSE(iterable_st);
@@ -6021,6 +7263,11 @@
}
TARGET(LOAD_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR);
@@ -6077,7 +7324,7 @@
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
self_or_null[0] = PyStackRef_NULL;
}
@@ -6089,7 +7336,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
@@ -6101,8 +7348,14 @@
}
TARGET(LOAD_ATTR_CLASS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_CLASS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6115,9 +7368,17 @@
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
+ if (!PyType_Check(owner_o)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
@@ -6140,8 +7401,14 @@
}
TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6154,16 +7421,28 @@
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR);
+ if (!PyType_Check(owner_o)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_TYPE_VERSION
{
uint32_t type_version = read_u32(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_CLASS
{
@@ -6185,8 +7464,14 @@
}
TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6198,17 +7483,33 @@
PyObject *getattribute = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
- DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
- DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
+ if (f->func_version != func_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(
@@ -6222,8 +7523,14 @@
}
TARGET(LOAD_ATTR_INSTANCE_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_INSTANCE_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6237,14 +7544,22 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_INSTANCE_VALUE
{
@@ -6252,10 +7567,18 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
@@ -6278,8 +7601,14 @@
}
TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_LAZY_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6293,7 +7622,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
@@ -6301,7 +7634,11 @@
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
/* This object has a __dict__, just not yet created */
- DEOPT_IF(dict != NULL, LOAD_ATTR);
+ if (dict != NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
@@ -6322,8 +7659,14 @@
}
TARGET(LOAD_ATTR_METHOD_NO_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_NO_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6337,7 +7680,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
@@ -6359,8 +7706,14 @@
}
TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_METHOD_WITH_VALUES;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6374,14 +7727,22 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_KEYS_VERSION
{
@@ -6389,7 +7750,11 @@
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
@@ -6410,8 +7775,14 @@
}
TARGET(LOAD_ATTR_MODULE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_MODULE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6425,11 +7796,19 @@
owner = stack_pointer[-1];
uint32_t dict_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR);
+ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
mod_keys = keys;
}
// _LOAD_ATTR_MODULE_FROM_KEYS
@@ -6440,11 +7819,19 @@
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
// Clear mod_keys from stack in case we need to deopt
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (!increfed) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
#else
Py_INCREF(attr_o);
@@ -6468,8 +7855,14 @@
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6482,7 +7875,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
@@ -6500,8 +7897,14 @@
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6514,14 +7917,22 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyDictValues *ivs = _PyObject_InlineValues(owner_o);
- DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR);
+ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_KEYS_VERSION
{
@@ -6529,7 +7940,11 @@
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
@@ -6545,8 +7960,14 @@
}
TARGET(LOAD_ATTR_PROPERTY) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_PROPERTY;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6555,7 +7976,11 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _GUARD_TYPE_VERSION
{
@@ -6563,7 +7988,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
@@ -6573,10 +8002,26 @@
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
- DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR);
- DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR);
- DEOPT_IF(code->co_argcount != 1, LOAD_ATTR);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
+ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (code->co_kwonlyargcount) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (code->co_argcount != 1) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
STAT_INC(LOAD_ATTR, hit);
new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame);
new_frame->localsplus[0] = owner;
@@ -6611,8 +8056,14 @@
}
TARGET(LOAD_ATTR_SLOT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_SLOT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6626,7 +8077,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _LOAD_ATTR_SLOT
{
@@ -6634,10 +8089,18 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
- DEOPT_IF(attr_o == NULL, LOAD_ATTR);
+ if (attr_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
- DEOPT_IF(!increfed, LOAD_ATTR);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
@@ -6657,8 +8120,14 @@
}
TARGET(LOAD_ATTR_WITH_HINT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_ATTR_WITH_HINT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
@@ -6673,14 +8142,22 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
// _CHECK_ATTR_WITH_HINT
{
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict_o == NULL, LOAD_ATTR);
+ if (dict_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
assert(PyDict_CheckExact((PyObject *)dict_o));
dict = dict_o;
}
@@ -6689,26 +8166,46 @@
uint16_t hint = read_u16(&this_instr[4].cache);
PyObject *attr_o;
if (!LOCK_OBJECT(dict)) {
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
attr_o = ep->me_value;
if (attr_o == NULL) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, LOAD_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(LOAD_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
+ JUMP_TO_PREDICTED(LOAD_ATTR);
+ }
}
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
@@ -6728,6 +8225,11 @@
}
TARGET(LOAD_BUILD_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_BUILD_CLASS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_BUILD_CLASS);
@@ -6737,14 +8239,14 @@
int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (bc_o == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
bc = PyStackRef_FromPyObjectSteal(bc_o);
stack_pointer[0] = bc;
@@ -6754,6 +8256,11 @@
}
TARGET(LOAD_COMMON_CONSTANT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_COMMON_CONSTANT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_COMMON_CONSTANT);
@@ -6776,6 +8283,11 @@
}
TARGET(LOAD_CONST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST);
@@ -6809,6 +8321,11 @@
}
TARGET(LOAD_CONST_IMMORTAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST_IMMORTAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST_IMMORTAL);
@@ -6824,6 +8341,11 @@
}
TARGET(LOAD_CONST_MORTAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_CONST_MORTAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST_MORTAL);
@@ -6838,6 +8360,11 @@
}
TARGET(LOAD_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_DEREF);
@@ -6848,7 +8375,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
value = PyStackRef_FromPyObjectSteal(value_o);
stack_pointer[0] = value;
@@ -6858,6 +8385,11 @@
}
TARGET(LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST);
@@ -6871,6 +8403,11 @@
}
TARGET(LOAD_FAST_AND_CLEAR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_AND_CLEAR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
@@ -6884,6 +8421,11 @@
}
TARGET(LOAD_FAST_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_CHECK;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_CHECK);
@@ -6896,7 +8438,7 @@
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
value = PyStackRef_DUP(value_s);
stack_pointer[0] = value;
@@ -6906,6 +8448,11 @@
}
TARGET(LOAD_FAST_LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FAST_LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
@@ -6923,6 +8470,11 @@
}
TARGET(LOAD_FROM_DICT_OR_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FROM_DICT_OR_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
@@ -6939,7 +8491,7 @@
int err = PyMapping_GetOptionalItem(class_dict, name, &value_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (!value_o) {
PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
@@ -6948,7 +8500,7 @@
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
}
stack_pointer += -1;
@@ -6964,6 +8516,11 @@
}
TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_FROM_DICT_OR_GLOBALS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
@@ -6977,7 +8534,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(mod_or_class_dict);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (v_o == NULL) {
if (PyDict_CheckExact(GLOBALS())
@@ -6999,7 +8556,7 @@
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
@@ -7011,7 +8568,7 @@
int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (v_o == NULL) {
/* namespace 2: builtins */
@@ -7019,7 +8576,7 @@
int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (v_o == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -7027,7 +8584,7 @@
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -7040,6 +8597,11 @@
}
TARGET(LOAD_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL);
@@ -7076,7 +8638,7 @@
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(*res)) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
// _PUSH_NULL_CONDITIONAL
@@ -7090,8 +8652,14 @@
}
TARGET(LOAD_GLOBAL_BUILTIN) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL_BUILTIN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
@@ -7103,18 +8671,34 @@
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
assert(DK_IS_UNICODE(keys));
}
// _GUARD_BUILTINS_VERSION_PUSH_KEYS
{
uint16_t version = read_u16(&this_instr[3].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
builtins_keys = keys;
assert(DK_IS_UNICODE(builtins_keys));
}
@@ -7123,10 +8707,18 @@
uint16_t index = read_u16(&this_instr[4].cache);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
- DEOPT_IF(!increfed, LOAD_GLOBAL);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#else
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -7145,8 +8737,14 @@
}
TARGET(LOAD_GLOBAL_MODULE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_GLOBAL_MODULE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
@@ -7158,9 +8756,17 @@
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
- DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version, LOAD_GLOBAL);
+ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
globals_keys = keys;
assert(DK_IS_UNICODE(globals_keys));
}
@@ -7170,10 +8776,18 @@
uint16_t index = read_u16(&this_instr[4].cache);
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
- DEOPT_IF(res_o == NULL, LOAD_GLOBAL);
+ if (res_o == NULL) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#if Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
- DEOPT_IF(!increfed, LOAD_GLOBAL);
+ if (!increfed) {
+ UPDATE_MISS_STATS(LOAD_GLOBAL);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL));
+ JUMP_TO_PREDICTED(LOAD_GLOBAL);
+ }
#else
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -7192,6 +8806,11 @@
}
TARGET(LOAD_LOCALS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_LOCALS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_LOCALS);
@@ -7202,7 +8821,7 @@
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
locals = PyStackRef_FromPyObjectNew(l);
stack_pointer[0] = locals;
@@ -7212,6 +8831,11 @@
}
TARGET(LOAD_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_NAME);
@@ -7221,7 +8845,7 @@
PyObject *v_o = _PyEval_LoadName(tstate, frame, name);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (v_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
v = PyStackRef_FromPyObjectSteal(v_o);
stack_pointer[0] = v;
@@ -7231,6 +8855,11 @@
}
TARGET(LOAD_SMALL_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SMALL_INT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SMALL_INT);
@@ -7245,6 +8874,11 @@
}
TARGET(LOAD_SPECIAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SPECIAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_SPECIAL);
@@ -7269,7 +8903,7 @@
Py_TYPE(owner_o)->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
self_or_null = self_or_null_o == NULL ?
@@ -7282,13 +8916,17 @@
}
TARGET(LOAD_SUPER_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR);
PREDICTED_LOAD_SUPER_ATTR:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
- opcode = LOAD_SUPER_ATTR;
_PyStackRef global_super_st;
_PyStackRef class_st;
_PyStackRef self_st;
@@ -7330,7 +8968,7 @@
PyStackRef_CLOSE(global_super_st);
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
// we make no attempt to optimize here; specializations should
@@ -7365,7 +9003,7 @@
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (super == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
stack_pointer += -3;
@@ -7375,7 +9013,7 @@
Py_DECREF(super);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
}
@@ -7391,6 +9029,13 @@
}
TARGET(LOAD_SUPER_ATTR_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR_ATTR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
@@ -7407,8 +9052,16 @@
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(!(oparg & 1));
- DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
- DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+ if (global_super != (PyObject *)&PySuper_Type) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
+ if (!PyType_Check(class)) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -7418,7 +9071,7 @@
PyStackRef_CLOSE(class_st);
PyStackRef_CLOSE(self_st);
if (attr == NULL) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
attr_st = PyStackRef_FromPyObjectSteal(attr);
stack_pointer[-3] = attr_st;
@@ -7428,6 +9081,13 @@
}
TARGET(LOAD_SUPER_ATTR_METHOD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = LOAD_SUPER_ATTR_METHOD;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
@@ -7445,8 +9105,16 @@
PyObject *class = PyStackRef_AsPyObjectBorrow(class_st);
PyObject *self = PyStackRef_AsPyObjectBorrow(self_st);
assert(oparg & 1);
- DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
- DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
+ if (global_super != (PyObject *)&PySuper_Type) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
+ if (!PyType_Check(class)) {
+ UPDATE_MISS_STATS(LOAD_SUPER_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR));
+ JUMP_TO_PREDICTED(LOAD_SUPER_ATTR);
+ }
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
@@ -7456,7 +9124,7 @@
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (attr_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (method_found) {
self_or_null = self_st; // transfer ownership
@@ -7481,6 +9149,11 @@
}
TARGET(MAKE_CELL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAKE_CELL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_CELL);
@@ -7489,7 +9162,7 @@
PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg));
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyStackRef tmp = GETLOCAL(oparg);
GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell);
@@ -7500,6 +9173,11 @@
}
TARGET(MAKE_FUNCTION) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAKE_FUNCTION;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_FUNCTION);
@@ -7517,7 +9195,7 @@
PyStackRef_CLOSE(codeobj_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (func_obj == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
@@ -7529,6 +9207,11 @@
}
TARGET(MAP_ADD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MAP_ADD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAP_ADD);
@@ -7550,7 +9233,7 @@
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
@@ -7558,6 +9241,11 @@
}
TARGET(MATCH_CLASS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_CLASS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_CLASS);
@@ -7586,7 +9274,7 @@
}
else {
if (_PyErr_Occurred(tstate)) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
// Error!
attrs = PyStackRef_None; // Failure!
@@ -7598,6 +9286,11 @@
}
TARGET(MATCH_KEYS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_KEYS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_KEYS);
@@ -7612,7 +9305,7 @@
PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (values_or_none_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
stack_pointer[0] = values_or_none;
@@ -7622,6 +9315,11 @@
}
TARGET(MATCH_MAPPING) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_MAPPING;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_MAPPING);
@@ -7637,6 +9335,11 @@
}
TARGET(MATCH_SEQUENCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = MATCH_SEQUENCE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_SEQUENCE);
@@ -7652,6 +9355,11 @@
}
TARGET(NOP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = NOP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOP);
@@ -7659,6 +9367,11 @@
}
TARGET(NOT_TAKEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = NOT_TAKEN;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOT_TAKEN);
@@ -7666,6 +9379,11 @@
}
TARGET(POP_EXCEPT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_EXCEPT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_EXCEPT);
@@ -7683,6 +9401,11 @@
}
TARGET(POP_ITER) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_ITER;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_ITER);
@@ -7695,8 +9418,14 @@
}
TARGET(POP_JUMP_IF_FALSE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_FALSE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
_PyStackRef cond;
@@ -7712,8 +9441,14 @@
}
TARGET(POP_JUMP_IF_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
_PyStackRef value;
@@ -7745,8 +9480,14 @@
}
TARGET(POP_JUMP_IF_NOT_NONE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_NOT_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
_PyStackRef value;
@@ -7778,8 +9519,14 @@
}
TARGET(POP_JUMP_IF_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_JUMP_IF_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
_PyStackRef cond;
@@ -7795,6 +9542,11 @@
}
TARGET(POP_TOP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = POP_TOP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_TOP);
@@ -7807,6 +9559,11 @@
}
TARGET(PUSH_EXC_INFO) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = PUSH_EXC_INFO;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_EXC_INFO);
@@ -7832,6 +9589,11 @@
}
TARGET(PUSH_NULL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = PUSH_NULL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
@@ -7844,8 +9606,14 @@
}
TARGET(RAISE_VARARGS) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RAISE_VARARGS;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
_PyStackRef *args;
@@ -7864,14 +9632,20 @@
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
- goto error;
+ JUMP_TO_LABEL(error);
}
TARGET(RERAISE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RERAISE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RERAISE);
_PyStackRef *values;
@@ -7897,7 +9671,7 @@
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
Py_DECREF(exc);
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@@ -7910,10 +9684,15 @@
monitor_reraise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
TARGET(RESERVED) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESERVED;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESERVED);
@@ -7923,6 +9702,11 @@
}
TARGET(RESUME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESUME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME);
@@ -7939,7 +9723,7 @@
_PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
stack_pointer = _PyFrame_GetStackPointer(frame);
if (bytecode == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
@@ -7963,7 +9747,7 @@
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
next_instr = this_instr;
DISPATCH();
@@ -7988,7 +9772,7 @@
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
}
@@ -7997,26 +9781,50 @@
}
TARGET(RESUME_CHECK) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RESUME_CHECK;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME_CHECK);
static_assert(0 == 0, "incorrect cache size");
#if defined(__EMSCRIPTEN__)
- DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
+ if (_Py_emscripten_signal_clock == 0) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
- DEOPT_IF(eval_breaker != version, RESUME);
+ if (eval_breaker != version) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
#ifdef Py_GIL_DISABLED
- DEOPT_IF(frame->tlbc_index !=
- ((_PyThreadStateImpl *)tstate)->tlbc_index, RESUME);
+ if (frame->tlbc_index !=
+ ((_PyThreadStateImpl *)tstate)->tlbc_index) {
+ UPDATE_MISS_STATS(RESUME);
+ assert(_PyOpcode_Deopt[opcode] == (RESUME));
+ JUMP_TO_PREDICTED(RESUME);
+ }
#endif
DISPATCH();
}
TARGET(RETURN_GENERATOR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RETURN_GENERATOR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
@@ -8027,7 +9835,7 @@
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (gen == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -8052,6 +9860,11 @@
}
TARGET(RETURN_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = RETURN_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_VALUE);
@@ -8080,6 +9893,11 @@
}
TARGET(SEND) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SEND;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND);
@@ -8159,7 +9977,7 @@
}
else {
PyStackRef_CLOSE(v);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
stack_pointer += -1;
@@ -8176,6 +9994,13 @@
}
TARGET(SEND_GEN) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SEND_GEN;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
@@ -8187,15 +10012,27 @@
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
- DEOPT_IF(tstate->interp->eval_frame, SEND);
+ if (tstate->interp->eval_frame) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
}
// _SEND_GEN_FRAME
{
v = stack_pointer[-1];
receiver = stack_pointer[-2];
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver);
- DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
- DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
+ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
+ if (gen->gi_frame_state >= FRAME_EXECUTING) {
+ UPDATE_MISS_STATS(SEND);
+ assert(_PyOpcode_Deopt[opcode] == (SEND));
+ JUMP_TO_PREDICTED(SEND);
+ }
STAT_INC(SEND, hit);
gen_frame = &gen->gi_iframe;
_PyFrame_StackPush(gen_frame, v);
@@ -8228,6 +10065,11 @@
}
TARGET(SETUP_ANNOTATIONS) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SETUP_ANNOTATIONS;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SETUP_ANNOTATIONS);
@@ -8237,21 +10079,21 @@
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
/* check if __annotations__ in locals()... */
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
- goto error;
+ JUMP_TO_LABEL(error);
}
if (ann_dict == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
ann_dict = PyDict_New();
stack_pointer = _PyFrame_GetStackPointer(frame);
if (ann_dict == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
@@ -8259,7 +10101,7 @@
Py_DECREF(ann_dict);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
}
else {
@@ -8271,6 +10113,11 @@
}
TARGET(SET_ADD) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_ADD;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_ADD);
@@ -8284,7 +10131,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -8292,6 +10139,11 @@
}
TARGET(SET_FUNCTION_ATTRIBUTE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_FUNCTION_ATTRIBUTE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
@@ -8316,6 +10168,11 @@
}
TARGET(SET_UPDATE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SET_UPDATE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_UPDATE);
@@ -8329,7 +10186,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(iterable);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -8337,6 +10194,11 @@
}
TARGET(STORE_ATTR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR);
@@ -8375,7 +10237,7 @@
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(owner);
if (err) {
- goto pop_2_error;
+ JUMP_TO_LABEL(pop_2_error);
}
}
stack_pointer += -2;
@@ -8384,8 +10246,14 @@
}
TARGET(STORE_ATTR_INSTANCE_VALUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_INSTANCE_VALUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
@@ -8398,11 +10266,19 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
- DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
+ if (!LOCK_OBJECT(owner_o)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
PyTypeObject *tp = Py_TYPE(owner_o);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UNLOCK_OBJECT(owner_o);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
}
// _GUARD_DORV_NO_DICT
@@ -8413,7 +10289,11 @@
if (_PyObject_GetManagedDict(owner_o) ||
!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) {
UNLOCK_OBJECT(owner_o);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
}
// _STORE_ATTR_INSTANCE_VALUE
@@ -8443,8 +10323,14 @@
}
TARGET(STORE_ATTR_SLOT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_SLOT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
@@ -8457,14 +10343,22 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
// _STORE_ATTR_SLOT
{
value = stack_pointer[-2];
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
- DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR);
+ if (!LOCK_OBJECT(owner_o)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
char *addr = (char *)owner_o + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
@@ -8481,8 +10375,14 @@
}
TARGET(STORE_ATTR_WITH_HINT) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_ATTR_WITH_HINT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
@@ -8495,7 +10395,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
// _STORE_ATTR_WITH_HINT
{
@@ -8504,12 +10408,24 @@
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
- DEOPT_IF(dict == NULL, STORE_ATTR);
- DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR);
+ if (dict == NULL) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
+ if (!LOCK_OBJECT(dict)) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
#ifdef Py_GIL_DISABLED
if (dict != _PyObject_GetManagedDict(owner_o)) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
#endif
assert(PyDict_CheckExact((PyObject *)dict));
@@ -8517,17 +10433,29 @@
if (hint >= (size_t)dict->ma_keys->dk_nentries ||
!DK_IS_UNICODE(dict->ma_keys)) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
PyObject *old_value = ep->me_value;
if (old_value == NULL) {
UNLOCK_OBJECT(dict);
- DEOPT_IF(true, STORE_ATTR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_ATTR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR));
+ JUMP_TO_PREDICTED(STORE_ATTR);
+ }
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value));
@@ -8548,6 +10476,11 @@
}
TARGET(STORE_DEREF) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_DEREF;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_DEREF);
@@ -8563,6 +10496,11 @@
}
TARGET(STORE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST);
@@ -8579,6 +10517,11 @@
}
TARGET(STORE_FAST_LOAD_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST_LOAD_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
@@ -8598,6 +10541,11 @@
}
TARGET(STORE_FAST_STORE_FAST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_FAST_STORE_FAST;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
@@ -8625,6 +10573,11 @@
}
TARGET(STORE_GLOBAL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_GLOBAL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_GLOBAL);
@@ -8636,7 +10589,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -8644,6 +10597,11 @@
}
TARGET(STORE_NAME) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_NAME;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_NAME);
@@ -8658,7 +10616,7 @@
"no locals found when storing %R", name);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(v);
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
if (PyDict_CheckExact(ns)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
@@ -8672,7 +10630,7 @@
}
PyStackRef_CLOSE(v);
if (err) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
@@ -8680,6 +10638,11 @@
}
TARGET(STORE_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SLICE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_SLICE);
@@ -8721,7 +10684,7 @@
PyStackRef_CLOSE(v);
PyStackRef_CLOSE(container);
if (err) {
- goto pop_4_error;
+ JUMP_TO_LABEL(pop_4_error);
}
}
stack_pointer += -4;
@@ -8730,6 +10693,11 @@
}
TARGET(STORE_SUBSCR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR);
@@ -8768,7 +10736,7 @@
PyStackRef_CLOSE(container);
PyStackRef_CLOSE(sub);
if (err) {
- goto pop_3_error;
+ JUMP_TO_LABEL(pop_3_error);
}
}
stack_pointer += -3;
@@ -8777,6 +10745,13 @@
}
TARGET(STORE_SUBSCR_DICT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR_DICT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
@@ -8789,7 +10764,11 @@
dict_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
- DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
+ if (!PyDict_CheckExact(dict)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
STAT_INC(STORE_SUBSCR, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
@@ -8802,12 +10781,19 @@
PyStackRef_CLOSE(dict_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
- goto error;
+ JUMP_TO_LABEL(error);
}
DISPATCH();
}
TARGET(STORE_SUBSCR_LIST_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = STORE_SUBSCR_LIST_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
@@ -8821,16 +10807,36 @@
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
- DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
- DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
+ if (!PyLong_CheckExact(sub)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
+ if (!PyList_CheckExact(list)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
// Ensure nonnegative, zero-or-one-digit ints.
- DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
+ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
- DEOPT_IF(!LOCK_OBJECT(list), STORE_SUBSCR);
+ if (!LOCK_OBJECT(list)) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
// Ensure index < len(list)
if (index >= PyList_GET_SIZE(list)) {
UNLOCK_OBJECT(list);
- DEOPT_IF(true, STORE_SUBSCR);
+ if (true) {
+ UPDATE_MISS_STATS(STORE_SUBSCR);
+ assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
+ JUMP_TO_PREDICTED(STORE_SUBSCR);
+ }
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
@@ -8848,6 +10854,11 @@
}
TARGET(SWAP) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = SWAP;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SWAP);
@@ -8863,6 +10874,11 @@
}
TARGET(TO_BOOL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL);
@@ -8896,7 +10912,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (err < 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = err ? PyStackRef_True : PyStackRef_False;
}
@@ -8905,8 +10921,14 @@
}
TARGET(TO_BOOL_ALWAYS_TRUE) {
- _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr;
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_ALWAYS_TRUE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
+ frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
@@ -8920,7 +10942,11 @@
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
- DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL);
+ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
}
// _REPLACE_WITH_TRUE
{
@@ -8933,6 +10959,13 @@
}
TARGET(TO_BOOL_BOOL) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_BOOL;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
@@ -8941,12 +10974,23 @@
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
- DEOPT_IF(!PyStackRef_BoolCheck(value), TO_BOOL);
+ if (!PyStackRef_BoolCheck(value)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
DISPATCH();
}
TARGET(TO_BOOL_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_INT;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
@@ -8957,7 +11001,11 @@
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyLong_CheckExact(value_o), TO_BOOL);
+ if (!PyLong_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value_o)) {
assert(_Py_IsImmortal(value_o));
@@ -8972,6 +11020,13 @@
}
TARGET(TO_BOOL_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
@@ -8982,7 +11037,11 @@
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyList_CheckExact(value_o), TO_BOOL);
+ if (!PyList_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
PyStackRef_CLOSE(value);
@@ -8991,6 +11050,13 @@
}
TARGET(TO_BOOL_NONE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_NONE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
@@ -9001,7 +11067,11 @@
/* Skip 2 cache entries */
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
- DEOPT_IF(!PyStackRef_IsNone(value), TO_BOOL);
+ if (!PyStackRef_IsNone(value)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
res = PyStackRef_False;
stack_pointer[-1] = res;
@@ -9009,6 +11079,13 @@
}
TARGET(TO_BOOL_STR) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = TO_BOOL_STR;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
@@ -9019,7 +11096,11 @@
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
- DEOPT_IF(!PyUnicode_CheckExact(value_o), TO_BOOL);
+ if (!PyUnicode_CheckExact(value_o)) {
+ UPDATE_MISS_STATS(TO_BOOL);
+ assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
+ JUMP_TO_PREDICTED(TO_BOOL);
+ }
STAT_INC(TO_BOOL, hit);
if (value_o == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value_o));
@@ -9035,6 +11116,11 @@
}
TARGET(UNARY_INVERT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_INVERT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_INVERT);
@@ -9046,7 +11132,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
@@ -9054,6 +11140,11 @@
}
TARGET(UNARY_NEGATIVE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_NEGATIVE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NEGATIVE);
@@ -9065,7 +11156,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(value);
if (res_o == NULL) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-1] = res;
@@ -9073,6 +11164,11 @@
}
TARGET(UNARY_NOT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNARY_NOT;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NOT);
@@ -9087,6 +11183,11 @@
}
TARGET(UNPACK_EX) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_EX;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNPACK_EX);
@@ -9100,7 +11201,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(seq);
if (res == 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
stack_pointer += (oparg & 0xFF) + (oparg >> 8);
assert(WITHIN_STACK_BOUNDS());
@@ -9108,6 +11209,11 @@
}
TARGET(UNPACK_SEQUENCE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE);
@@ -9144,7 +11250,7 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(seq);
if (res == 0) {
- goto pop_1_error;
+ JUMP_TO_LABEL(pop_1_error);
}
}
stack_pointer += -1 + oparg;
@@ -9153,6 +11259,13 @@
}
TARGET(UNPACK_SEQUENCE_LIST) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_LIST;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
@@ -9163,11 +11276,23 @@
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE);
+ if (!PyList_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (!LOCK_OBJECT(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
- DEOPT_IF(true, UNPACK_SEQUENCE);
+ if (true) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq_o);
@@ -9182,6 +11307,13 @@
}
TARGET(UNPACK_SEQUENCE_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
@@ -9192,8 +11324,16 @@
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg, UNPACK_SEQUENCE);
+ if (!PyTuple_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (PyTuple_GET_SIZE(seq_o) != oparg) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
@@ -9206,6 +11346,13 @@
}
TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = UNPACK_SEQUENCE_TWO_TUPLE;
+ (void)(opcode);
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
@@ -9217,8 +11364,16 @@
seq = stack_pointer[-1];
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
- DEOPT_IF(!PyTuple_CheckExact(seq_o), UNPACK_SEQUENCE);
- DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2, UNPACK_SEQUENCE);
+ if (!PyTuple_CheckExact(seq_o)) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
+ if (PyTuple_GET_SIZE(seq_o) != 2) {
+ UPDATE_MISS_STATS(UNPACK_SEQUENCE);
+ assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
+ JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
+ }
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
@@ -9231,6 +11386,11 @@
}
TARGET(WITH_EXCEPT_START) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = WITH_EXCEPT_START;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(WITH_EXCEPT_START);
@@ -9275,7 +11435,7 @@
(3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- goto error;
+ JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
@@ -9285,6 +11445,11 @@
}
TARGET(YIELD_VALUE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode;
+ #endif
+ opcode = YIELD_VALUE;
+ (void)(opcode);
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(YIELD_VALUE);
@@ -9331,6 +11496,7 @@
}
/* END INSTRUCTIONS */
+#ifndef Py_TAIL_CALL_INTERP
#if USE_COMPUTED_GOTOS
_unknown_opcode:
#else
@@ -9340,44 +11506,46 @@
next_instr points the current instruction without TARGET(). */
opcode = next_instr->op.code;
_PyErr_Format(tstate, PyExc_SystemError,
- "%U:%d: unknown opcode %d",
- _PyFrame_GetCode(frame)->co_filename,
- PyUnstable_InterpreterFrame_GetLine(frame),
- opcode);
- goto error;
+ "%U:%d: unknown opcode %d",
+ _PyFrame_GetCode(frame)->co_filename,
+ PyUnstable_InterpreterFrame_GetLine(frame),
+ opcode);
+JUMP_TO_LABEL(error);
+
}
/* This should never be reached. Every opcode should end with DISPATCH()
or goto error. */
Py_UNREACHABLE();
+#endif /* Py_TAIL_CALL_INTERP */
/* BEGIN LABELS */
- pop_4_error:
+ LABEL(pop_4_error)
{
STACK_SHRINK(4);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_3_error:
+ LABEL(pop_3_error)
{
STACK_SHRINK(3);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_2_error:
+ LABEL(pop_2_error)
{
STACK_SHRINK(2);
- goto error;
+ JUMP_TO_LABEL(error);
}
- pop_1_error:
+ LABEL(pop_1_error)
{
STACK_SHRINK(1);
- goto error;
+ JUMP_TO_LABEL(error);
}
- error:
+ LABEL(error)
{
/* Double-check exception status. */
#ifdef NDEBUG
@@ -9407,10 +11575,10 @@
_PyEval_MonitorRaise(tstate, frame, next_instr-1);
stack_pointer = _PyFrame_GetStackPointer(frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
- exception_unwind:
+ LABEL(exception_unwind)
{
/* STACK SPILLED */
/* We can't use frame->instr_ptr here, as RERAISE may have set it */
@@ -9427,7 +11595,7 @@
PyStackRef_XCLOSE(ref);
}
monitor_unwind(tstate, frame, next_instr-1);
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
assert(STACK_LEVEL() >= level);
_PyStackRef *new_top = _PyFrame_Stackbase(frame) + level;
@@ -9440,7 +11608,7 @@
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
PyObject *lasti = PyLong_FromLong(frame_lasti);
if (lasti == NULL) {
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
}
@@ -9453,7 +11621,7 @@
next_instr = _PyFrame_GetBytecode(frame) + handler;
int err = monitor_handled(tstate, frame, next_instr, exc);
if (err < 0) {
- goto exception_unwind;
+ JUMP_TO_LABEL(exception_unwind);
}
/* Resume normal execution */
#ifdef LLTRACE
@@ -9462,10 +11630,13 @@
}
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
+ #ifdef Py_TAIL_CALL_INTERP
+ int opcode;
+ #endif
DISPATCH();
}
- exit_unwind:
+ LABEL(exit_unwind)
{
/* STACK SPILLED */
assert(_PyErr_Occurred(tstate));
@@ -9484,15 +11655,15 @@
}
next_instr = frame->instr_ptr;
stack_pointer = _PyFrame_GetStackPointer(frame);
- goto error;
+ JUMP_TO_LABEL(error);
}
- start_frame:
+ LABEL(start_frame)
{
/* STACK SPILLED */
int too_deep = _Py_EnterRecursivePy(tstate);
if (too_deep) {
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
next_instr = frame->instr_ptr;
#ifdef LLTRACE
@@ -9500,7 +11671,7 @@
int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS());
frame->lltrace = lltrace;
if (lltrace < 0) {
- goto exit_unwind;
+ JUMP_TO_LABEL(exit_unwind);
}
}
#endif
@@ -9512,6 +11683,9 @@
assert(!_PyErr_Occurred(tstate));
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
+ #ifdef Py_TAIL_CALL_INTERP
+ int opcode;
+ #endif
DISPATCH();
}