diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 217 | ||||
| -rw-r--r-- | Python/codegen.c | 27 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 306 | ||||
| -rw-r--r-- | Python/flowgraph.c | 6 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 633 | ||||
| -rw-r--r-- | Python/instrumentation.c | 5 | ||||
| -rw-r--r-- | Python/opcode_targets.h | 18 | ||||
| -rw-r--r-- | Python/optimizer.c | 5 | ||||
| -rw-r--r-- | Python/optimizer_analysis.c | 4 | ||||
| -rw-r--r-- | Python/optimizer_bytecodes.c | 30 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 104 | ||||
| -rw-r--r-- | Python/specialize.c | 108 |
12 files changed, 720 insertions, 743 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0ef767..398f1d5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -348,8 +348,8 @@ dummy_func( DECREF_INPUTS(); } - pure inst(PUSH_NULL, (-- res)) { - res = PyStackRef_NULL; + pure inst(PUSH_NULL, (-- null)) { + null = PyStackRef_NULL; } no_save_ip inst(END_FOR, (value -- )) { @@ -1658,7 +1658,7 @@ dummy_func( specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1669,11 +1669,10 @@ dummy_func( } // res[1] because we need a pointer to res to pass it to _PyEval_LoadGlobalStackRef - op(_LOAD_GLOBAL, ( -- res[1], null if (oparg & 1))) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + op(_LOAD_GLOBAL, ( -- res[1])) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); ERROR_IF(PyStackRef_IsNull(*res), error); - null = PyStackRef_NULL; } macro(LOAD_GLOBAL) = @@ -1711,7 +1710,7 @@ dummy_func( assert(DK_IS_UNICODE(builtins_keys)); } - op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { + op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res)) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); DEAD(globals_keys); @@ -1725,10 +1724,9 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } - op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { + op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res)) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); DEAD(builtins_keys); @@ -1742,7 +1740,6 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } macro(LOAD_GLOBAL_MODULE) = @@ -2012,17 +2009,27 @@ dummy_func( GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } + inst(INSTRUMENTED_LOAD_SUPER_METHOD, (unused/1 -- )) { + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_METHOD); + } + family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { LOAD_SUPER_ATTR_ATTR, - LOAD_SUPER_ATTR_METHOD, + }; + + + family(LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { + LOAD_SUPER_METHOD_METHOD, }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -2030,12 +2037,24 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { + specializing op(_SPECIALIZE_LOAD_SUPER_METHOD, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + + tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr)) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2049,7 +2068,7 @@ dummy_func( // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _Py_call_instrumentation_exc2( @@ -2072,12 +2091,13 @@ dummy_func( Py_DECREF(super); ERROR_IF(attr_o == NULL, error); attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; } macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; + macro(LOAD_SUPER_METHOD) = _SPECIALIZE_LOAD_SUPER_METHOD + _LOAD_SUPER_ATTR + PUSH_NULL; + - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st)) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); @@ -2093,7 +2113,7 @@ dummy_func( attr_st = PyStackRef_FromPyObjectSteal(attr); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { + inst(LOAD_SUPER_METHOD_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); @@ -2132,17 +2152,20 @@ dummy_func( LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, LOAD_ATTR_PROPERTY, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - LOAD_ATTR_METHOD_WITH_VALUES, - LOAD_ATTR_METHOD_NO_DICT, - LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; + family(LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { + LOAD_METHOD_WITH_VALUES, + LOAD_METHOD_NO_DICT, + LOAD_METHOD_LAZY_DICT, + }; + specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -2152,50 +2175,67 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + specializing op(_SPECIALIZE_LOAD_METHOD, (counter/1, owner -- owner)) { + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_LoadMethod(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + + op(_LOAD_METHOD, (owner -- attr, self_or_null)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - DEAD(owner); - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN */ - DECREF_INPUTS(); - ERROR_IF(attr_o == NULL, error); - self_or_null = PyStackRef_NULL; - } + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + DEAD(owner); } else { - /* Classic, pushes one value. */ - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ DECREF_INPUTS(); ERROR_IF(attr_o == NULL, error); - /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } + op(_LOAD_ATTR, (owner -- attr)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + DECREF_INPUTS(); + ERROR_IF(attr_o == NULL, error); + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + unused/8 + _LOAD_ATTR; + + macro(LOAD_METHOD) = + _SPECIALIZE_LOAD_METHOD + + unused/8 + + _LOAD_METHOD; + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -2220,7 +2260,7 @@ dummy_func( DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); } - split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -2233,7 +2273,6 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2254,7 +2293,7 @@ dummy_func( mod_keys = keys; } - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr)) { assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; @@ -2272,7 +2311,6 @@ dummy_func( attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } @@ -2292,7 +2330,7 @@ dummy_func( dict = dict_o; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr)) { PyObject *attr_o; if (!LOCK_OBJECT(dict)) { POP_INPUT(dict); @@ -2304,7 +2342,7 @@ dummy_func( POP_INPUT(dict); DEOPT_IF(true); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); POP_INPUT(dict); @@ -2326,7 +2364,6 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); DEAD(dict); - null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2337,7 +2374,7 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); @@ -2350,7 +2387,6 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2368,11 +2404,10 @@ dummy_func( EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version); } - split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2389,7 +2424,6 @@ dummy_func( _LOAD_ATTR_CLASS; op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) { - assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -2412,10 +2446,8 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - - assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -2429,7 +2461,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -3348,8 +3380,7 @@ dummy_func( DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); } - split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { - assert(oparg & 1); + op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3359,15 +3390,14 @@ dummy_func( DEAD(owner); } - macro(LOAD_ATTR_METHOD_WITH_VALUES) = + macro(LOAD_METHOD_WITH_VALUES) = unused/1 + _GUARD_TYPE_VERSION + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + _GUARD_KEYS_VERSION + - _LOAD_ATTR_METHOD_WITH_VALUES; + _LOAD_METHOD_WITH_VALUES; - op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { - assert(oparg & 1); + op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3377,14 +3407,13 @@ dummy_func( DEAD(owner); } - macro(LOAD_ATTR_METHOD_NO_DICT) = + macro(LOAD_METHOD_NO_DICT) = unused/1 + _GUARD_TYPE_VERSION + unused/2 + - _LOAD_ATTR_METHOD_NO_DICT; + _LOAD_METHOD_NO_DICT; - op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr, unused if (0))) { - assert((oparg & 1) == 0); + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); @@ -3398,8 +3427,7 @@ dummy_func( _GUARD_KEYS_VERSION + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { - assert((oparg & 1) == 0); + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3420,8 +3448,7 @@ dummy_func( DEOPT_IF(dict != NULL); } - op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { - assert(oparg & 1); + op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -3430,12 +3457,12 @@ dummy_func( DEAD(owner); } - macro(LOAD_ATTR_METHOD_LAZY_DICT) = + macro(LOAD_METHOD_LAZY_DICT) = unused/1 + _GUARD_TYPE_VERSION + _CHECK_ATTR_METHOD_LAZY_DICT + unused/1 + - _LOAD_ATTR_METHOD_LAZY_DICT; + _LOAD_METHOD_LAZY_DICT; // Cache layout: counter/1, func_version/2 // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! @@ -4537,7 +4564,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { + op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in -- func, unused, tuple, kwargs_out)) { PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -4561,7 +4588,7 @@ dummy_func( } } - op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { + op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); // DICT_MERGE is called before this opcode if there are kwargs. @@ -4695,11 +4722,16 @@ dummy_func( LLTRACE_RESUME_FRAME(); } - inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { + inst(BUILD_SLICE, (args[oparg] -- slice)) { + assert(oparg == 2 || oparg == 3); + _PyStackRef start = args[0]; + _PyStackRef stop = args[1]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); - + PyObject * step_o = NULL; + if (oparg == 3) { + step_o = PyStackRef_AsPyObjectBorrow(args[2]); + } PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); ERROR_IF(slice_o == NULL, error); @@ -5041,27 +5073,25 @@ dummy_func( DEOPT_IF(func->func_version != func_version); } - tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { + tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL); Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); - null = PyStackRef_NULL; } - tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { + tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res)) { PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL); Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); - null = PyStackRef_NULL; } - tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { + tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); @@ -5072,7 +5102,6 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; DECREF_INPUTS(); } diff --git a/Python/codegen.c b/Python/codegen.c index 61707ba..b3f8455 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -348,8 +348,6 @@ codegen_addop_o(compiler *c, location loc, RETURN_IF_ERROR_IN_SCOPE((C), ret); \ } while (0) -#define LOAD_METHOD -1 -#define LOAD_SUPER_METHOD -2 #define LOAD_ZERO_SUPER_ATTR -3 #define LOAD_ZERO_SUPER_METHOD -4 @@ -366,20 +364,11 @@ codegen_addop_name(compiler *c, location loc, if (arg < 0) { return ERROR; } - if (opcode == LOAD_ATTR) { - arg <<= 1; - } - if (opcode == LOAD_METHOD) { - opcode = LOAD_ATTR; - arg <<= 1; - arg |= 1; - } if (opcode == LOAD_SUPER_ATTR) { arg <<= 2; arg |= 2; } if (opcode == LOAD_SUPER_METHOD) { - opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 3; } @@ -388,7 +377,7 @@ codegen_addop_name(compiler *c, location loc, arg <<= 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { - opcode = LOAD_SUPER_ATTR; + opcode = LOAD_SUPER_METHOD; arg <<= 2; arg |= 1; } @@ -3165,9 +3154,6 @@ codegen_nameop(compiler *c, location loc, assert(op); Py_DECREF(mangled); - if (op == LOAD_GLOBAL) { - arg <<= 1; - } ADDOP_I(c, loc, op, arg); return SUCCESS; @@ -4108,7 +4094,10 @@ ex_call: } assert(have_dict); } - ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); + if (nkwelts == 0) { + ADDOP(c, loc, PUSH_NULL); + } + ADDOP(c, loc, CALL_FUNCTION_EX); return SUCCESS; } @@ -4841,8 +4830,10 @@ codegen_async_with(compiler *c, stmt_ty s, int pos) SETUP_WITH E <code to store to VAR> or POP_TOP <code for BLOCK> - LOAD_CONST (None, None, None) - CALL_FUNCTION_EX 0 + LOAD_CONST None + LOAD_CONST None + LOAD_CONST None + CALL 3 JUMP EXIT E: WITH_EXCEPT_START (calls EXPR.__exit__) POP_JUMP_IF_TRUE T: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e2eaca2..0490731 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -406,9 +406,9 @@ } case _PUSH_NULL: { - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; + _PyStackRef null; + null = PyStackRef_NULL; + stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1957,17 +1957,14 @@ case _LOAD_GLOBAL: { _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR(); - null = PyStackRef_NULL; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2033,8 +2030,6 @@ case _LOAD_GLOBAL_MODULE_FROM_KEYS: { PyDictKeysObject *globals_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); @@ -2056,10 +2051,8 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2067,8 +2060,6 @@ case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { PyDictKeysObject *builtins_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); @@ -2090,10 +2081,8 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2519,6 +2508,8 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ + /* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 because it is instrumented */ + case _LOAD_SUPER_ATTR_ATTR: { _PyStackRef self_st; _PyStackRef class_st; @@ -2556,7 +2547,7 @@ break; } - case _LOAD_SUPER_ATTR_METHOD: { + case _LOAD_SUPER_METHOD_METHOD: { _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -2605,58 +2596,62 @@ break; } - case _LOAD_ATTR: { + case _LOAD_METHOD: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; + _PyStackRef self_or_null; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) JUMP_TO_ERROR(); - self_or_null = PyStackRef_NULL; - } + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ PyStackRef_CLOSE(owner); if (attr_o == NULL) JUMP_TO_ERROR(); - /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer[0] = self_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } + case _LOAD_ATTR: { + _PyStackRef owner; + _PyStackRef attr; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-1] = attr; + break; + } + case _GUARD_TYPE_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2704,42 +2699,9 @@ break; } - case _LOAD_ATTR_INSTANCE_VALUE_0: { - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; - owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - break; - } - - case _LOAD_ATTR_INSTANCE_VALUE_1: { + case _LOAD_ATTR_INSTANCE_VALUE: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2760,17 +2722,11 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); break; } - /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ - case _CHECK_ATTR_MODULE_PUSH_KEYS: { _PyStackRef owner; PyDictKeysObject *mod_keys; @@ -2799,8 +2755,6 @@ PyDictKeysObject *mod_keys; _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits; owner = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); @@ -2828,12 +2782,8 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } @@ -2860,7 +2810,6 @@ PyDictObject *dict; _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); dict = (PyDictObject *)stack_pointer[-1].bits; owner = stack_pointer[-2]; @@ -2883,7 +2832,7 @@ JUMP_TO_JUMP_TARGET(); } } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); stack_pointer += -1; @@ -2916,20 +2865,16 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-2] = attr; - if (oparg & 1) stack_pointer[-1] = null; - stack_pointer += -1 + (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_SLOT_0: { + case _LOAD_ATTR_SLOT: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2949,47 +2894,11 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_SLOT_1: { - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ - case _CHECK_ATTR_CLASS: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -3007,50 +2916,24 @@ break; } - case _LOAD_ATTR_CLASS_0: { + case _LOAD_ATTR_CLASS: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } - case _LOAD_ATTR_CLASS_1: { - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - (void)null; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; - PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - stack_pointer[0] = null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ - case _LOAD_ATTR_PROPERTY_FRAME: { _PyStackRef owner; _PyInterpreterFrame *new_frame; - oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *fget = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -4056,14 +3939,12 @@ break; } - case _LOAD_ATTR_METHOD_WITH_VALUES: { + case _LOAD_METHOD_WITH_VALUES: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - oparg = CURRENT_OPARG(); + _PyStackRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4077,14 +3958,12 @@ break; } - case _LOAD_ATTR_METHOD_NO_DICT: { + case _LOAD_METHOD_NO_DICT: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - oparg = CURRENT_OPARG(); + _PyStackRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4101,10 +3980,8 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _PyStackRef owner; _PyStackRef attr; - oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -4116,10 +3993,8 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _PyStackRef owner; _PyStackRef attr; - oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4143,14 +4018,12 @@ break; } - case _LOAD_ATTR_METHOD_LAZY_DICT: { + case _LOAD_METHOD_LAZY_DICT: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - oparg = CURRENT_OPARG(); + _PyStackRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -5644,15 +5517,14 @@ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ case _MAKE_CALLARGS_A_TUPLE: { - _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef kwargs_in; _PyStackRef callargs; _PyStackRef func; _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; - oparg = CURRENT_OPARG(); - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + _PyStackRef kwargs_out; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -5675,8 +5547,8 @@ PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - stack_pointer[-1 - (oparg & 1)] = tuple; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; + stack_pointer[-2] = tuple; + stack_pointer[-1] = kwargs_out; break; } @@ -5753,25 +5625,27 @@ } case _BUILD_SLICE: { - _PyStackRef step = PyStackRef_NULL; - _PyStackRef stop; - _PyStackRef start; + _PyStackRef *args; _PyStackRef slice; oparg = CURRENT_OPARG(); - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + args = &stack_pointer[-oparg]; + assert(oparg == 2 || oparg == 3); + _PyStackRef start = args[0]; + _PyStackRef stop = args[1]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject * step_o = NULL; + if (oparg == 3) { + step_o = PyStackRef_AsPyObjectBorrow(args[2]); + } PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (slice_o == NULL) JUMP_TO_ERROR(); slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6138,8 +6012,6 @@ case _LOAD_GLOBAL_MODULE: { _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -6150,18 +6022,14 @@ } Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); - null = PyStackRef_NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS: { _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -6172,10 +6040,8 @@ } Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); - null = PyStackRef_NULL; stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6183,8 +6049,6 @@ case _LOAD_ATTR_MODULE: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; - oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -6200,12 +6064,8 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 24561c1..3f8d9db 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1836,12 +1836,6 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) INSTR_SET_OP0(inst, NOP); } break; - case LOAD_GLOBAL: - if (nextop == PUSH_NULL && (oparg & 1) == 0) { - INSTR_SET_OP1(inst, LOAD_GLOBAL, oparg | 1); - INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); - } - break; case COMPARE_OP: if (nextop == TO_BOOL) { INSTR_SET_OP0(inst, NOP); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dc90f75..ec3397e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -835,28 +835,30 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef start; - _PyStackRef stop; - _PyStackRef step = PyStackRef_NULL; + _PyStackRef *args; _PyStackRef slice; - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + args = &stack_pointer[-oparg]; + assert(oparg == 2 || oparg == 3); + _PyStackRef start = args[0]; + _PyStackRef stop = args[1]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject * step_o = NULL; + if (oparg == 3) { + step_o = PyStackRef_AsPyObjectBorrow(args[2]); + } PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - PyStackRef_CLOSE(start); - PyStackRef_CLOSE(stop); - PyStackRef_XCLOSE(step); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } if (slice_o == NULL) { - stack_pointer += -2 - ((oparg == 3) ? 1 : 0); + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1710,18 +1712,18 @@ (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in = PyStackRef_NULL; + _PyStackRef kwargs_in; _PyStackRef tuple; - _PyStackRef kwargs_out = PyStackRef_NULL; + _PyStackRef kwargs_out; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st = PyStackRef_NULL; + _PyStackRef kwargs_st; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } - callargs = stack_pointer[-1 - (oparg & 1)]; - func = stack_pointer[-3 - (oparg & 1)]; + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -1763,8 +1765,8 @@ assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -1807,7 +1809,7 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2 - (oparg & 1); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -1828,8 +1830,8 @@ assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-1 - (oparg & 1)] = callargs_st; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1839,11 +1841,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) { - stack_pointer += -3 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - goto error; - } + if (result_o == NULL) goto pop_4_error; result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -1851,19 +1849,19 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 2 + (oparg & 1); + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-3 - (oparg & 1)] = result; - stack_pointer += -2 - (oparg & 1); + stack_pointer[-4] = result; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4850,6 +4848,18 @@ GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } + TARGET(INSTRUMENTED_LOAD_SUPER_METHOD) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + (void)this_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_METHOD); + /* Skip 1 cache entry */ + // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we + // don't want to specialize instrumented instructions + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + GO_TO_INSTRUCTION(LOAD_SUPER_METHOD); + } + TARGET(INSTRUMENTED_NOT_TAKEN) { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5307,7 +5317,6 @@ (void)this_instr; _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -5315,7 +5324,7 @@ (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -5329,50 +5338,15 @@ /* Skip 8 cache entries */ // _LOAD_ATTR { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - } - else { - /* Classic, pushes one value. */ - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - /* We need to define self_or_null on all paths */ - self_or_null = PyStackRef_NULL; - } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5383,7 +5357,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -5401,13 +5374,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5418,7 +5387,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -5442,13 +5410,9 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5464,7 +5428,6 @@ uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -5477,7 +5440,7 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -5495,7 +5458,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5527,143 +5489,10 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self = PyStackRef_NULL; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5675,7 +5504,6 @@ _PyStackRef owner; PyDictKeysObject *mod_keys; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE_PUSH_KEYS { @@ -5708,14 +5536,10 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5739,7 +5563,6 @@ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -5784,7 +5607,6 @@ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -5818,7 +5640,6 @@ // _LOAD_ATTR_PROPERTY_FRAME { PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -5866,7 +5687,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5890,14 +5710,10 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5909,7 +5725,6 @@ _PyStackRef owner; PyDictObject *dict; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5939,7 +5754,7 @@ UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR); @@ -5957,14 +5772,10 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6265,14 +6076,13 @@ _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; - _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -6289,15 +6099,13 @@ // _LOAD_GLOBAL { res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; - null = PyStackRef_NULL; } - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6309,7 +6117,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -6344,11 +6151,9 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6360,7 +6165,6 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; _PyStackRef res; - _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION_PUSH_KEYS { @@ -6387,11 +6191,9 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - null = PyStackRef_NULL; } stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6416,6 +6218,198 @@ DISPATCH(); } + TARGET(LOAD_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD); + PREDICTED(LOAD_METHOD); + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + // _SPECIALIZE_LOAD_METHOD + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadMethod(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_METHOD + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *attr_o; + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_METHOD_LAZY_DICT) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_METHOD); + } + /* Skip 1 cache entry */ + // _LOAD_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_METHOD_NO_DICT) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); + } + /* Skip 2 cache entries */ + // _LOAD_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_METHOD_WITH_VALUES) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD); + } + // _LOAD_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -6495,7 +6489,6 @@ _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; - _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR { class_st = stack_pointer[-2]; @@ -6503,11 +6496,10 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -6521,7 +6513,7 @@ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( @@ -6541,7 +6533,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + if (opcode >= MIN_INSTRUMENTED_OPCODE) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6574,11 +6566,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); - null = PyStackRef_NULL; } stack_pointer[0] = attr; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6618,10 +6608,111 @@ DISPATCH(); } - TARGET(LOAD_SUPER_ATTR_METHOD) { + TARGET(LOAD_SUPER_METHOD) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + INSTRUCTION_STATS(LOAD_SUPER_METHOD); + PREDICTED(LOAD_SUPER_METHOD); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null; + // _SPECIALIZE_LOAD_SUPER_METHOD + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } + } + } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL + { + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_SUPER_METHOD_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_METHOD_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -6636,8 +6727,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7002,9 +7093,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; + _PyStackRef null; + null = PyStackRef_NULL; + stack_pointer[0] = null; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 17e5346..b24e288 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -81,6 +81,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL, + [LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL, [RESUME] = -1, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, @@ -126,6 +128,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [INSTRUMENTED_LOAD_SUPER_METHOD] = LOAD_SUPER_METHOD, [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN, }; @@ -164,6 +167,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD, + [INSTRUMENTED_LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD, [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index cb6c33f..2e35ae7 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -3,6 +3,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -15,8 +16,8 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_ITER, &&TARGET_RESERVED, + &&TARGET_GET_ITER, &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_INTERPRETER_EXIT, @@ -51,7 +52,6 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, @@ -89,10 +89,12 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_DEREF, &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_GLOBAL, + &&TARGET_LOAD_METHOD, &&TARGET_LOAD_NAME, &&TARGET_LOAD_SMALL_INT, &&TARGET_LOAD_SPECIAL, &&TARGET_LOAD_SUPER_ATTR, + &&TARGET_LOAD_SUPER_METHOD, &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, &&TARGET_MATCH_CLASS, @@ -146,8 +148,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -198,9 +198,6 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, @@ -211,8 +208,11 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_CONST_MORTAL, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_METHOD_LAZY_DICT, + &&TARGET_LOAD_METHOD_NO_DICT, + &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_ATTR_METHOD, + &&TARGET_LOAD_SUPER_METHOD_METHOD, &&TARGET_RESUME_CHECK, &&TARGET_SEND_GEN, &&TARGET_STORE_ATTR_INSTANCE_VALUE, @@ -233,11 +233,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_INSTRUMENTED_LOAD_SUPER_METHOD, &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, diff --git a/Python/optimizer.c b/Python/optimizer.c index 9beb472..b91caf1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1274,10 +1274,7 @@ uop_optimize( for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; - if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { - buffer[pc].opcode = opcode + 1 + (oparg & 1); - } - else if (oparg < _PyUop_Replication[opcode]) { + if (oparg < _PyUop_Replication[opcode]) { buffer[pc].opcode = opcode + oparg + 1; } else if (is_terminator(&buffer[pc])) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b9ac30e..05f9592 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -109,10 +109,10 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) return NULL; } if (_Py_IsImmortal(res)) { - inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; + inst->opcode = _LOAD_CONST_INLINE_BORROW; } else { - inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; + inst->opcode = _LOAD_CONST_INLINE; } inst->operand0 = (uint64_t)res; return res; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 881a607..af834b6 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -528,9 +528,8 @@ dummy_func(void) { top_out = top_in; } - op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)offset; (void)owner; } @@ -553,15 +552,19 @@ dummy_func(void) { } } - op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + op(_LOAD_ATTR, (owner -- attr)) { + (void)owner; + attr = sym_new_not_null(ctx); + } + + op(_LOAD_METHOD, (owner -- attr, self_or_null)) { (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); } - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) { (void)index; - null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. @@ -589,41 +592,38 @@ dummy_func(void) { (void)owner; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) { attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)hint; (void)owner; (void)dict; } - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)index; (void)owner; } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)descr; (void)owner; } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -819,7 +819,7 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fa0b4ed..cb4651a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -928,12 +928,9 @@ case _LOAD_GLOBAL: { JitOptSymbol **res; - JitOptSymbol *null = NULL; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); - null = sym_new_null(ctx); - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -966,25 +963,15 @@ case _LOAD_GLOBAL_MODULE_FROM_KEYS: { JitOptSymbol *res; - JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); - null = sym_new_null(ctx); stack_pointer[-1] = res; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { JitOptSymbol *res; - JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); - null = sym_new_null(ctx); stack_pointer[-1] = res; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1109,6 +1096,8 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 */ + case _LOAD_SUPER_ATTR_ATTR: { JitOptSymbol *attr_st; attr_st = sym_new_not_null(ctx); @@ -1118,7 +1107,7 @@ break; } - case _LOAD_SUPER_ATTR_METHOD: { + case _LOAD_SUPER_METHOD_METHOD: { JitOptSymbol *attr; JitOptSymbol *self_or_null; attr = sym_new_not_null(ctx); @@ -1130,21 +1119,31 @@ break; } - case _LOAD_ATTR: { + case _LOAD_METHOD: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self_or_null = NULL; + JitOptSymbol *self_or_null; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); + stack_pointer[0] = self_or_null; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } + case _LOAD_ATTR: { + JitOptSymbol *owner; + JitOptSymbol *attr; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + case _GUARD_TYPE_VERSION: { JitOptSymbol *owner; owner = stack_pointer[-1]; @@ -1182,17 +1181,12 @@ case _LOAD_ATTR_INSTANCE_VALUE: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)offset; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1230,11 +1224,9 @@ case _LOAD_ATTR_MODULE_FROM_KEYS: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *null = NULL; owner = stack_pointer[-2]; uint16_t index = (uint16_t)this_instr->operand0; (void)index; - null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. @@ -1243,8 +1235,7 @@ assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; stack_pointer[-2] = attr; - if (oparg & 1) stack_pointer[-1] = null; - stack_pointer += -1 + (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { @@ -1254,7 +1245,7 @@ else { this_instr->opcode = _LOAD_ATTR_MODULE; } - stack_pointer += 1 - (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } if (attr == NULL) { @@ -1262,8 +1253,7 @@ attr = sym_new_not_null(ctx); } stack_pointer[-2] = attr; - if (oparg & 1) stack_pointer[-1] = null; - stack_pointer += -1 + (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1284,18 +1274,15 @@ JitOptSymbol *dict; JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *null = NULL; dict = stack_pointer[-1]; owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)hint; (void)owner; (void)dict; stack_pointer[-2] = attr; - if (oparg & 1) stack_pointer[-1] = null; - stack_pointer += -1 + (oparg & 1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1303,17 +1290,12 @@ case _LOAD_ATTR_SLOT: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)index; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1324,17 +1306,12 @@ case _LOAD_ATTR_CLASS: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); (void)descr; (void)owner; stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1720,10 +1697,10 @@ break; } - case _LOAD_ATTR_METHOD_WITH_VALUES: { + case _LOAD_METHOD_WITH_VALUES: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self = NULL; + JitOptSymbol *self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1736,10 +1713,10 @@ break; } - case _LOAD_ATTR_METHOD_NO_DICT: { + case _LOAD_METHOD_NO_DICT: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self = NULL; + JitOptSymbol *self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1770,10 +1747,10 @@ break; } - case _LOAD_ATTR_METHOD_LAZY_DICT: { + case _LOAD_METHOD_LAZY_DICT: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self = NULL; + JitOptSymbol *self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -2262,11 +2239,11 @@ case _MAKE_CALLARGS_A_TUPLE: { JitOptSymbol *tuple; - JitOptSymbol *kwargs_out = NULL; + JitOptSymbol *kwargs_out; tuple = sym_new_not_null(ctx); kwargs_out = sym_new_not_null(ctx); - stack_pointer[-1 - (oparg & 1)] = tuple; - if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; + stack_pointer[-2] = tuple; + stack_pointer[-1] = kwargs_out; break; } @@ -2315,8 +2292,8 @@ case _BUILD_SLICE: { JitOptSymbol *slice; slice = sym_new_not_null(ctx); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + stack_pointer[-oparg] = slice; + stack_pointer += 1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2639,37 +2616,26 @@ case _LOAD_GLOBAL_MODULE: { JitOptSymbol *res; - JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); - null = sym_new_null(ctx); stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS: { JitOptSymbol *res; - JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); - null = sym_new_null(ctx); stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_MODULE: { JitOptSymbol *attr; - JitOptSymbol *null = NULL; attr = sym_new_not_null(ctx); - null = sym_new_null(ctx); stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index fa02234..eb59902 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -804,7 +804,7 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; + uint8_t load_code = load_method ? LOAD_SUPER_METHOD_METHOD : LOAD_SUPER_ATTR_ATTR; specialize(instr, load_code); return; fail: @@ -1109,7 +1109,7 @@ instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) static int do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, bool shadow, uint32_t shared_keys_version, - DescriptorClassification kind, PyObject *descr, unsigned int tp_version) + DescriptorClassification kind, PyObject *descr, unsigned int tp_version, bool load_method) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); @@ -1117,17 +1117,16 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } - uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); return -1; case METHOD: { if (shadow) { goto try_instance; } - if (oparg & 1) { + if (load_method) { if (specialize_attr_loadclassattr(owner, instr, name, descr, tp_version, kind, true, shared_keys_version)) { @@ -1137,7 +1136,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; } } - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } case PROPERTY: @@ -1146,28 +1145,28 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* assert(Py_TYPE(descr) == &PyProperty_Type); PyObject *fget = ((_PyPropertyObject *)descr)->prop_get; if (fget == NULL) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); return -1; } if (!Py_IS_TYPE(fget, &PyFunction_Type)) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION); return -1; } - if (!function_check_args(fget, 1, LOAD_ATTR)) { + if (load_method) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_ATTR_METHOD); return -1; } - if (oparg & 1) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + if (!function_check_args(fget, 1, LOAD_ATTR)) { return -1; } /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } #ifdef Py_GIL_DISABLED if (!_PyObject_HasDeferredRefcount(fget)) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); return -1; } #endif @@ -1180,6 +1179,10 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } case OBJECT_SLOT: { + if (load_method) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + return -1; + } PyMemberDescrObject *member = (PyMemberDescrObject *)descr; struct PyMemberDef *dmem = member->d_member; Py_ssize_t offset = dmem->offset; @@ -1204,6 +1207,10 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } case DUNDER_CLASS: { + if (load_method) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + return -1; + } Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; @@ -1212,16 +1219,20 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return 0; } case OTHER_SLOT: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); return -1; case MUTABLE: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); return -1; case GETSET_OVERRIDDEN: - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); + SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { + if (load_method) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + return -1; + } #ifndef Py_GIL_DISABLED // In free-threaded builds it's possible for tp_getattro to change // after the call to analyze_descriptor. That is fine: the version @@ -1233,10 +1244,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* if (!function_check_args(descr, 2, LOAD_ATTR)) { return -1; } - if (oparg & 1) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); - return -1; - } uint32_t version = function_get_version(descr, LOAD_ATTR); if (version == 0) { return -1; @@ -1270,7 +1277,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* if (shadow) { goto try_instance; } - if ((oparg & 1) == 0) { + if (!load_method) { if (specialize_attr_loadclassattr(owner, instr, name, descr, tp_version, kind, false, shared_keys_version)) { @@ -1287,6 +1294,10 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } Py_UNREACHABLE(); try_instance: + if (load_method) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + return -1; + } if (specialize_dict_access(owner, instr, type, kind, name, tp_version, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { @@ -1296,7 +1307,7 @@ try_instance: } static int -specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, bool load_method) { // 0 is not a valid version uint32_t shared_keys_version = 0; @@ -1305,7 +1316,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na unsigned int tp_version = 0; PyTypeObject *type = Py_TYPE(owner); DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); - int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version, load_method); Py_XDECREF(descr); return result; } @@ -1333,7 +1344,40 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_class_load_attr(owner, instr, name); } else { - fail = specialize_instance_load_attr(owner, instr, name); + fail = specialize_instance_load_attr(owner, instr, name, false); + } + + if (fail) { + unspecialize(instr); + } +} + +void +_Py_Specialize_LoadMethod(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) +{ + PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); + + assert(ENABLE_SPECIALIZATION_FT); + assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); + PyTypeObject *type = Py_TYPE(owner); + bool fail; + if (!_PyType_IsReady(type)) { + // We *might* not really need this check, but we inherited it from + // PyObject_GenericGetAttr and friends... and this way we still do the + // right thing if someone forgets to call PyType_Ready(type): + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + fail = true; + } + else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + fail = true; + } + else if (PyType_Check(owner)) { + SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); + fail = true; + } + else { + fail = specialize_instance_load_attr(owner, instr, name, true); } if (fail) { @@ -1575,7 +1619,7 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, #ifdef Py_GIL_DISABLED if (!_PyObject_HasDeferredRefcount(descr)) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); return 0; } #endif @@ -1587,11 +1631,11 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); #endif if (shared_keys_version == 0) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } write_u32(cache->keys_version, shared_keys_version); - specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + specialize(instr, is_method ? LOAD_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; @@ -1601,17 +1645,17 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, else { dictoffset = owner_cls->tp_dictoffset; if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); return 0; } } if (dictoffset == 0) { - specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + specialize(instr, is_method ? LOAD_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); } else if (is_method) { PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); if (dict) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } /* Cache entries must be unsigned values, so we offset the @@ -1620,10 +1664,10 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, dictoffset -= MANAGED_DICT_OFFSET; assert(((uint16_t)dictoffset) == dictoffset); cache->dict_offset = (uint16_t)dictoffset; - specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT); + specialize(instr, LOAD_METHOD_LAZY_DICT); } else { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); return 0; } } |