diff options
| -rw-r--r-- | Lib/test/test_generated_cases.py | 11 | ||||
| -rw-r--r-- | Python/bytecodes.c | 352 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 73 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 132 | ||||
| -rw-r--r-- | Tools/cases_generator/analysis.py | 26 | ||||
| -rw-r--r-- | Tools/cases_generator/generate_cases.py | 4 | ||||
| -rw-r--r-- | Tools/cases_generator/instructions.py | 16 | ||||
| -rw-r--r-- | Tools/cases_generator/stacking.py | 16 |
8 files changed, 315 insertions, 315 deletions
diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index b5eaf82..5971d2e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -239,16 +239,22 @@ class TestGeneratedCases(unittest.TestCase): def test_predictions_and_eval_breaker(self): input = """ - inst(OP1, (--)) { + inst(OP1, (arg -- rest)) { } inst(OP3, (arg -- res)) { - DEOPT_IF(xxx, OP1); + DEOPT_IF(xxx); CHECK_EVAL_BREAKER(); } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ output = """ TARGET(OP1) { PREDICTED(OP1); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + PyObject *arg; + PyObject *rest; + arg = stack_pointer[-1]; + stack_pointer[-1] = rest; DISPATCH(); } @@ -371,6 +377,7 @@ class TestGeneratedCases(unittest.TestCase): } TARGET(OP) { + PREDICTED(OP); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f7681bd..035629d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -41,8 +41,6 @@ #include "ceval_macros.h" /* Flow control macros */ -#define DEOPT_IF(cond, instname) ((void)0) -#define ERROR_IF(cond, labelname) ((void)0) #define GO_TO_INSTRUCTION(instname) ((void)0) #define inst(name, ...) case name: @@ -153,13 +151,12 @@ dummy_func( inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)); } inst(INSTRUMENTED_RESUME, (--)) { @@ -325,12 +322,12 @@ dummy_func( } inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -343,7 +340,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -351,13 +348,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + DEOPT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -373,7 +370,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -397,8 +394,8 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) { @@ -433,8 +430,8 @@ dummy_func( _GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) { @@ -469,8 +466,8 @@ dummy_func( _GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); } op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { @@ -494,7 +491,7 @@ dummy_func( _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -574,13 +571,13 @@ dummy_func( } inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -590,14 +587,14 @@ dummy_func( } inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyUnicode_CheckExact(str)); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -605,13 +602,13 @@ dummy_func( } inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -621,7 +618,7 @@ dummy_func( } inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); if (res == NULL) { @@ -636,19 +633,19 @@ dummy_func( } inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); + DEOPT_IF(cached == NULL); assert(PyFunction_Check(cached)); PyFunctionObject *getitem = (PyFunctionObject *)cached; uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(getitem->func_version != cached_version); PyCodeObject *code = (PyCodeObject *)getitem->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(BINARY_SUBSCR, hit); Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); @@ -693,14 +690,14 @@ dummy_func( } inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -712,7 +709,7 @@ dummy_func( } inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1009,11 +1006,10 @@ dummy_func( } inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; STACK_SHRINK(1); @@ -1194,8 +1190,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1204,8 +1200,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1215,8 +1211,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq)); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1412,15 +1408,15 @@ dummy_func( op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } op(_GUARD_BUILTINS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } @@ -1428,7 +1424,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1438,7 +1434,7 @@ dummy_func( PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1763,8 +1759,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1774,8 +1770,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1864,22 +1860,20 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1894,15 +1888,15 @@ dummy_func( unused/5; // Skip over rest of cache inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != type_version); assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1912,27 +1906,27 @@ dummy_func( inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); uint16_t hint = index; - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1942,7 +1936,7 @@ dummy_func( op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1957,9 +1951,8 @@ dummy_func( inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner)); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -1972,18 +1965,18 @@ dummy_func( inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); @@ -1997,17 +1990,17 @@ dummy_func( inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -2025,7 +2018,7 @@ dummy_func( op(_GUARD_DORV_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { @@ -2052,30 +2045,30 @@ dummy_func( inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2093,7 +2086,7 @@ dummy_func( op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { @@ -2140,8 +2133,8 @@ dummy_func( } inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -2155,10 +2148,10 @@ dummy_func( // Similar to COMPARE_OP_FLOAT inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2174,8 +2167,8 @@ dummy_func( // Similar to COMPARE_OP_FLOAT, but for ==, != only inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2492,7 +2485,7 @@ dummy_func( } op(_ITER_CHECK_LIST, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type); } op(_ITER_JUMP_LIST, (iter -- iter)) { @@ -2548,7 +2541,7 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type); } op(_ITER_JUMP_TUPLE, (iter -- iter)) { @@ -2605,7 +2598,7 @@ dummy_func( op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type); } op(_ITER_JUMP_RANGE, (iter -- iter)) { @@ -2647,10 +2640,10 @@ dummy_func( _ITER_NEXT_RANGE; inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; _PyFrame_StackPush(gen_frame, Py_None); @@ -2790,16 +2783,13 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -2839,15 +2829,12 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); @@ -2858,7 +2845,7 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2869,12 +2856,12 @@ dummy_func( inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -3002,8 +2989,8 @@ dummy_func( } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { @@ -3016,22 +3003,22 @@ dummy_func( } op(_CHECK_PEP_523, (--)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL)); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { @@ -3094,24 +3081,24 @@ dummy_func( _PUSH_FRAME; inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; } - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; assert(func->func_defaults); assert(PyTuple_CheckExact(func->func_defaults)); int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); assert(defcount <= code->co_argcount); int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(argcount > code->co_argcount); + DEOPT_IF(argcount < min_args); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); for (int i = 0; i < argcount; i++) { @@ -3130,9 +3117,9 @@ dummy_func( inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(null != NULL); PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); @@ -3141,8 +3128,8 @@ dummy_func( inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); @@ -3154,8 +3141,8 @@ dummy_func( inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); @@ -3172,15 +3159,15 @@ dummy_func( * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)next_instr; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(code->co_argcount != oparg+1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { @@ -3233,9 +3220,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3254,9 +3241,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + DEOPT_IF(total_args != 1); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -3282,8 +3269,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3314,9 +3301,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -3341,9 +3327,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + DEOPT_IF(callable != interp->callable_cache.len); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3365,9 +3351,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable != interp->callable_cache.isinstance); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3388,9 +3374,9 @@ dummy_func( inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append); assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); + DEOPT_IF(!PyList_Check(self)); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already @@ -3411,13 +3397,13 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3442,12 +3428,12 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = @@ -3471,13 +3457,13 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + DEOPT_IF(meth->ml_flags != METH_NOARGS); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3502,11 +3488,11 @@ dummy_func( } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 55a03c9..662de57 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,8 +13,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); break; } @@ -236,8 +235,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); + DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); break; } @@ -294,8 +293,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); + DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); break; } @@ -352,8 +351,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); + DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); break; } @@ -1189,8 +1188,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1198,8 +1197,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1211,7 +1210,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1229,7 +1228,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1679,7 +1678,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); break; } @@ -1689,9 +1688,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); break; } @@ -1703,7 +1700,7 @@ uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1722,7 +1719,7 @@ uint16_t index = (uint16_t)operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1738,7 +1735,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); break; } @@ -1770,7 +1767,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION_STORE); break; } @@ -2114,7 +2111,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); break; } @@ -2159,7 +2156,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); break; } @@ -2205,7 +2202,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); break; } @@ -2300,9 +2297,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); break; } @@ -2312,8 +2307,7 @@ uint32_t keys_version = (uint32_t)operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); break; } @@ -2360,8 +2354,8 @@ PyObject *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); break; } @@ -2382,7 +2376,7 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); break; } @@ -2392,11 +2386,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)operand; - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); break; } @@ -2405,8 +2399,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); + DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); break; } @@ -2671,8 +2665,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2701d41..96d9d9d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -32,8 +32,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); DISPATCH(); } @@ -391,8 +390,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT { @@ -416,8 +415,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT { @@ -441,8 +440,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT { @@ -466,8 +465,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT { @@ -491,8 +490,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT { @@ -516,8 +515,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { @@ -541,8 +540,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { @@ -565,15 +564,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE { _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -1294,8 +1293,7 @@ receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; @@ -1806,8 +1804,8 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_MODULE @@ -1816,7 +1814,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1836,16 +1834,16 @@ { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_BUILTINS @@ -1854,7 +1852,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -2403,23 +2401,21 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2510,14 +2506,14 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&next_instr[3].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2539,8 +2535,7 @@ PyObject *descr = read_obj(&next_instr[5].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -2627,13 +2622,13 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; @@ -2716,7 +2711,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT value = stack_pointer[-2]; @@ -3297,7 +3292,7 @@ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -3339,7 +3334,7 @@ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -3382,7 +3377,7 @@ iter = stack_pointer[-1]; { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -3585,23 +3580,20 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&next_instr[3].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -3631,7 +3623,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { @@ -3664,12 +3656,9 @@ DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); Py_DECREF(owner); @@ -3845,14 +3834,14 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -3868,18 +3857,18 @@ callable = func; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3939,25 +3928,25 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -4328,8 +4317,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 91dcba8..7bbc924 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -26,7 +26,7 @@ RESERVED_WORDS = { "co_names": "Use FRAME_CO_NAMES.", } -RE_PREDICTED = r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$" +RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" class Analyzer: @@ -187,16 +187,23 @@ class Analyzer: Raises SystemExit if there is an error. """ self.analyze_macros_and_pseudos() - self.find_predictions() self.map_families() + self.mark_predictions() self.check_families() - def find_predictions(self) -> None: - """Find the instructions that need PREDICTED() labels.""" + def mark_predictions(self) -> None: + """Mark the instructions that need PREDICTED() labels.""" + # Start with family heads + for family in self.families.values(): + if family.name in self.instrs: + self.instrs[family.name].predicted = True + if family.name in self.macro_instrs: + self.macro_instrs[family.name].predicted = True + # Also look for GO_TO_INSTRUCTION() calls for instr in self.instrs.values(): targets: set[str] = set() for line in instr.block_text: - if m := re.match(RE_PREDICTED, line): + if m := re.match(RE_GO_TO_INSTR, line): targets.add(m.group(1)) for target in targets: if target_instr := self.instrs.get(target): @@ -225,11 +232,18 @@ class Analyzer: ) else: member_instr.family = family - elif not self.macro_instrs.get(member): + if member_mac := self.macro_instrs.get(member): + assert member_mac.family is None, (member, member_mac.family.name) + member_mac.family = family + if not member_instr and not member_mac: self.error( f"Unknown instruction {member!r} referenced in family {family.name!r}", family, ) + # A sanctioned exception: + # This opcode is a member of the family but it doesn't pass the checks. + if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"): + mac.family = self.families.get("BINARY_OP") def check_families(self) -> None: """Check each family: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 9192d10..01ab83b 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -781,9 +781,7 @@ class Generator(Analyzer): case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr( - mac, self.out, self.families.get(mac.name) - ) + stacking.write_macro_instr(mac, self.out) case parsing.Pseudo(): pass case _: diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6fbf7d9..5384bbe 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -144,7 +144,8 @@ class Instruction: out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: """Write the instruction body.""" # Write cache effect variable declarations and initializations @@ -207,6 +208,14 @@ class Instruction: ) else: out.write_raw(f"{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): + space, cond = m.groups() + target = family.name if family else self.name + out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") + elif "DEOPT" in line: + filename = context.owner.filename + lineno = context.owner.tokens[context.begin].line + print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): out.reset_lineno() space = extra + m.group(1) @@ -244,7 +253,8 @@ class AbstractInstruction(Instruction): out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: pass @@ -268,7 +278,9 @@ class MacroInstruction: macro: parsing.Macro parts: MacroParts cache_offset: int + # Set later predicted: bool = False + family: parsing.Family | None = None @dataclasses.dataclass diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 1f9fda6..bba2db8 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -351,14 +351,13 @@ def write_single_instr( out, tier, 0, + instr.family, ) except AssertionError as err: raise AssertionError(f"Error writing instruction {instr.name}") from err -def write_macro_instr( - mac: MacroInstruction, out: Formatter, family: Family | None -) -> None: +def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: parts = [ part for part in mac.parts @@ -368,9 +367,11 @@ def write_macro_instr( with out.block(f"TARGET({mac.name})"): if mac.predicted: out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, family, mac.cache_offset) + out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: - next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) + next_instr_is_set = write_components( + parts, out, TIER_ONE, mac.cache_offset, mac.family + ) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err if not parts[-1].instr.always_exits: @@ -386,6 +387,7 @@ def write_components( out: Formatter, tier: Tiers, cache_offset: int, + family: Family | None, ) -> bool: managers = get_managers(parts) @@ -454,10 +456,10 @@ def write_components( assert_no_pokes(managers) if len(parts) == 1: - mgr.instr.write_body(out, 0, mgr.active_caches, tier) + mgr.instr.write_body(out, 0, mgr.active_caches, tier, family) else: with out.block(""): - mgr.instr.write_body(out, -4, mgr.active_caches, tier) + mgr.instr.write_body(out, -4, mgr.active_caches, tier, family) if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits: # Adjust the stack to its final depth, *then* write the |