diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 217 | ||||
| -rw-r--r-- | Python/codegen.c | 27 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 306 | ||||
| -rw-r--r-- | Python/flowgraph.c | 6 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 633 | ||||
| -rw-r--r-- | Python/instrumentation.c | 5 | ||||
| -rw-r--r-- | Python/opcode_targets.h | 18 | ||||
| -rw-r--r-- | Python/optimizer.c | 5 | ||||
| -rw-r--r-- | Python/optimizer_analysis.c | 4 | ||||
| -rw-r--r-- | Python/optimizer_bytecodes.c | 30 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 104 | ||||
| -rw-r--r-- | Python/specialize.c | 108 |
12 files changed, 743 insertions, 720 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c78d496..b70ed7f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -348,8 +348,8 @@ dummy_func( DECREF_INPUTS(); } - pure inst(PUSH_NULL, (-- null)) { - null = PyStackRef_NULL; + pure inst(PUSH_NULL, (-- res)) { + res = PyStackRef_NULL; } no_save_ip inst(END_FOR, (value -- )) { @@ -1654,7 +1654,7 @@ dummy_func( specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); @@ -1665,10 +1665,11 @@ dummy_func( } // res[1] because we need a pointer to res to pass it to _PyEval_LoadGlobalStackRef - op(_LOAD_GLOBAL, ( -- res[1])) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + op(_LOAD_GLOBAL, ( -- res[1], null if (oparg & 1))) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); ERROR_IF(PyStackRef_IsNull(*res), error); + null = PyStackRef_NULL; } macro(LOAD_GLOBAL) = @@ -1706,7 +1707,7 @@ dummy_func( assert(DK_IS_UNICODE(builtins_keys)); } - op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res)) { + op(_LOAD_GLOBAL_MODULE_FROM_KEYS, (index/1, globals_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); DEAD(globals_keys); @@ -1720,9 +1721,10 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } - op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res)) { + op(_LOAD_GLOBAL_BUILTINS_FROM_KEYS, (index/1, builtins_keys: PyDictKeysObject* -- res, null if (oparg & 1))) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); DEAD(builtins_keys); @@ -1736,6 +1738,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } macro(LOAD_GLOBAL_MODULE) = @@ -2009,27 +2012,17 @@ dummy_func( GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } - inst(INSTRUMENTED_LOAD_SUPER_METHOD, (unused/1 -- )) { - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_METHOD); - } - family(LOAD_SUPER_ATTR, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { LOAD_SUPER_ATTR_ATTR, - }; - - - family(LOAD_SUPER_METHOD, INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR) = { - LOAD_SUPER_METHOD_METHOD, + LOAD_SUPER_ATTR_METHOD, }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -2037,24 +2030,12 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - specializing op(_SPECIALIZE_LOAD_SUPER_METHOD, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - - tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr)) { + tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -2068,7 +2049,7 @@ dummy_func( // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _Py_call_instrumentation_exc2( @@ -2091,13 +2072,12 @@ dummy_func( Py_DECREF(super); ERROR_IF(attr_o == NULL, error); attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; } macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; - macro(LOAD_SUPER_METHOD) = _SPECIALIZE_LOAD_SUPER_METHOD + _LOAD_SUPER_ATTR + PUSH_NULL; - - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st)) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super_st, class_st, self_st -- attr_st, unused if (0))) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); @@ -2113,7 +2093,7 @@ dummy_func( attr_st = PyStackRef_FromPyObjectSteal(attr); } - inst(LOAD_SUPER_METHOD_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super_st, class_st, self_st -- attr, self_or_null)) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); @@ -2152,20 +2132,17 @@ dummy_func( LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, LOAD_ATTR_PROPERTY, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + LOAD_ATTR_METHOD_WITH_VALUES, + LOAD_ATTR_METHOD_NO_DICT, + LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - family(LOAD_METHOD, INLINE_CACHE_ENTRIES_LOAD_ATTR) = { - LOAD_METHOD_WITH_VALUES, - LOAD_METHOD_NO_DICT, - LOAD_METHOD_LAZY_DICT, - }; - specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); @@ -2175,67 +2152,50 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - specializing op(_SPECIALIZE_LOAD_METHOD, (counter/1, owner -- owner)) { - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_LoadMethod(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - - op(_LOAD_METHOD, (owner -- attr, self_or_null)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + DEAD(owner); + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - DEAD(owner); + DECREF_INPUTS(); + ERROR_IF(attr_o == NULL, error); + self_or_null = PyStackRef_NULL; + } } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ + /* Classic, pushes one value. */ + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); DECREF_INPUTS(); ERROR_IF(attr_o == NULL, error); + /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); } - op(_LOAD_ATTR, (owner -- attr)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - DECREF_INPUTS(); - ERROR_IF(attr_o == NULL, error); - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + unused/8 + _LOAD_ATTR; - - macro(LOAD_METHOD) = - _SPECIALIZE_LOAD_METHOD + - unused/8 + - _LOAD_METHOD; - op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); @@ -2260,7 +2220,7 @@ dummy_func( DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); } - op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { + split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -2273,6 +2233,7 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2293,7 +2254,7 @@ dummy_func( mod_keys = keys; } - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr)) { + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr, null if (oparg & 1))) { assert(mod_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; @@ -2311,6 +2272,7 @@ dummy_func( attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } @@ -2330,7 +2292,7 @@ dummy_func( dict = dict_o; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr)) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) { PyObject *attr_o; if (!LOCK_OBJECT(dict)) { POP_INPUT(dict); @@ -2342,7 +2304,7 @@ dummy_func( POP_INPUT(dict); DEOPT_IF(true); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); POP_INPUT(dict); @@ -2364,6 +2326,7 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); DEAD(dict); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2374,7 +2337,7 @@ dummy_func( _LOAD_ATTR_WITH_HINT + unused/5; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { + split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); @@ -2387,6 +2350,7 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2404,10 +2368,11 @@ dummy_func( EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version); } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { + split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2424,6 +2389,7 @@ dummy_func( _LOAD_ATTR_CLASS; op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) { + assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -2446,8 +2412,10 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused)) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -2461,7 +2429,7 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -3380,7 +3348,8 @@ dummy_func( DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); } - op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { + split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { + assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3390,14 +3359,15 @@ dummy_func( DEAD(owner); } - macro(LOAD_METHOD_WITH_VALUES) = + macro(LOAD_ATTR_METHOD_WITH_VALUES) = unused/1 + _GUARD_TYPE_VERSION + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + _GUARD_KEYS_VERSION + - _LOAD_METHOD_WITH_VALUES; + _LOAD_ATTR_METHOD_WITH_VALUES; - op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { + assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3407,13 +3377,14 @@ dummy_func( DEAD(owner); } - macro(LOAD_METHOD_NO_DICT) = + macro(LOAD_ATTR_METHOD_NO_DICT) = unused/1 + _GUARD_TYPE_VERSION + unused/2 + - _LOAD_METHOD_NO_DICT; + _LOAD_ATTR_METHOD_NO_DICT; - op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr, unused if (0))) { + assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); @@ -3427,7 +3398,8 @@ dummy_func( _GUARD_KEYS_VERSION + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr, unused if (0))) { + assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3448,7 +3420,8 @@ dummy_func( DEOPT_IF(dict != NULL); } - op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { + assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -3457,12 +3430,12 @@ dummy_func( DEAD(owner); } - macro(LOAD_METHOD_LAZY_DICT) = + macro(LOAD_ATTR_METHOD_LAZY_DICT) = unused/1 + _GUARD_TYPE_VERSION + _CHECK_ATTR_METHOD_LAZY_DICT + unused/1 + - _LOAD_METHOD_LAZY_DICT; + _LOAD_ATTR_METHOD_LAZY_DICT; // Cache layout: counter/1, func_version/2 // CALL_INTRINSIC_1/2, CALL_KW, and CALL_FUNCTION_EX aren't members! @@ -4564,7 +4537,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in -- func, unused, tuple, kwargs_out)) { + op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs_in if (oparg & 1) -- func, unused, tuple, kwargs_out if (oparg & 1))) { PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -4588,7 +4561,7 @@ dummy_func( } } - op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st -- result)) { + op(_DO_CALL_FUNCTION_EX, (func_st, unused, callargs_st, kwargs_st if (oparg & 1) -- result)) { PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); // DICT_MERGE is called before this opcode if there are kwargs. @@ -4722,16 +4695,11 @@ dummy_func( LLTRACE_RESUME_FRAME(); } - inst(BUILD_SLICE, (args[oparg] -- slice)) { - assert(oparg == 2 || oparg == 3); - _PyStackRef start = args[0]; - _PyStackRef stop = args[1]; + inst(BUILD_SLICE, (start, stop, step if (oparg == 3) -- slice)) { PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject * step_o = NULL; - if (oparg == 3) { - step_o = PyStackRef_AsPyObjectBorrow(args[2]); - } + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); DECREF_INPUTS(); ERROR_IF(slice_o == NULL, error); @@ -5074,25 +5042,27 @@ dummy_func( DEOPT_IF(func->func_version != func_version); } - tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res)) { + tier2 op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL); Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; } - tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res)) { + tier2 op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); PyObject *res_o = entries[index].me_value; DEOPT_IF(res_o == NULL); Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; } - tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr)) { + tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); @@ -5103,6 +5073,7 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; DECREF_INPUTS(); } diff --git a/Python/codegen.c b/Python/codegen.c index b3f8455..61707ba 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -348,6 +348,8 @@ codegen_addop_o(compiler *c, location loc, RETURN_IF_ERROR_IN_SCOPE((C), ret); \ } while (0) +#define LOAD_METHOD -1 +#define LOAD_SUPER_METHOD -2 #define LOAD_ZERO_SUPER_ATTR -3 #define LOAD_ZERO_SUPER_METHOD -4 @@ -364,11 +366,20 @@ codegen_addop_name(compiler *c, location loc, if (arg < 0) { return ERROR; } + if (opcode == LOAD_ATTR) { + arg <<= 1; + } + if (opcode == LOAD_METHOD) { + opcode = LOAD_ATTR; + arg <<= 1; + arg |= 1; + } if (opcode == LOAD_SUPER_ATTR) { arg <<= 2; arg |= 2; } if (opcode == LOAD_SUPER_METHOD) { + opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 3; } @@ -377,7 +388,7 @@ codegen_addop_name(compiler *c, location loc, arg <<= 2; } if (opcode == LOAD_ZERO_SUPER_METHOD) { - opcode = LOAD_SUPER_METHOD; + opcode = LOAD_SUPER_ATTR; arg <<= 2; arg |= 1; } @@ -3154,6 +3165,9 @@ codegen_nameop(compiler *c, location loc, assert(op); Py_DECREF(mangled); + if (op == LOAD_GLOBAL) { + arg <<= 1; + } ADDOP_I(c, loc, op, arg); return SUCCESS; @@ -4094,10 +4108,7 @@ ex_call: } assert(have_dict); } - if (nkwelts == 0) { - ADDOP(c, loc, PUSH_NULL); - } - ADDOP(c, loc, CALL_FUNCTION_EX); + ADDOP_I(c, loc, CALL_FUNCTION_EX, nkwelts > 0); return SUCCESS; } @@ -4830,10 +4841,8 @@ codegen_async_with(compiler *c, stmt_ty s, int pos) SETUP_WITH E <code to store to VAR> or POP_TOP <code for BLOCK> - LOAD_CONST None - LOAD_CONST None - LOAD_CONST None - CALL 3 + LOAD_CONST (None, None, None) + CALL_FUNCTION_EX 0 JUMP EXIT E: WITH_EXCEPT_START (calls EXPR.__exit__) POP_JUMP_IF_TRUE T: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d2da9a8..d336f73 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -406,9 +406,9 @@ } case _PUSH_NULL: { - _PyStackRef null; - null = PyStackRef_NULL; - stack_pointer[0] = null; + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; @@ -1953,14 +1953,17 @@ case _LOAD_GLOBAL: { _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR(); - stack_pointer += 1; + null = PyStackRef_NULL; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2026,6 +2029,8 @@ case _LOAD_GLOBAL_MODULE_FROM_KEYS: { PyDictKeysObject *globals_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys); @@ -2047,8 +2052,10 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2056,6 +2063,8 @@ case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { PyDictKeysObject *builtins_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys); @@ -2077,8 +2086,10 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2508,8 +2519,6 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 because it is instrumented */ - case _LOAD_SUPER_ATTR_ATTR: { _PyStackRef self_st; _PyStackRef class_st; @@ -2547,7 +2556,7 @@ break; } - case _LOAD_SUPER_METHOD_METHOD: { + case _LOAD_SUPER_ATTR_METHOD: { _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -2596,62 +2605,58 @@ break; } - case _LOAD_METHOD: { + case _LOAD_ATTR: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self_or_null; + _PyStackRef self_or_null = PyStackRef_NULL; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) JUMP_TO_ERROR(); + self_or_null = PyStackRef_NULL; + } } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(owner); if (attr_o == NULL) JUMP_TO_ERROR(); + /* We need to define self_or_null on all paths */ self_or_null = PyStackRef_NULL; } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR: { - _PyStackRef owner; - _PyStackRef attr; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) JUMP_TO_ERROR(); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-1] = attr; - break; - } - case _GUARD_TYPE_VERSION: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2699,9 +2704,42 @@ break; } - case _LOAD_ATTR_INSTANCE_VALUE: { + case _LOAD_ATTR_INSTANCE_VALUE_0: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_1: { _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2722,11 +2760,17 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } + /* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */ + case _CHECK_ATTR_MODULE_PUSH_KEYS: { _PyStackRef owner; PyDictKeysObject *mod_keys; @@ -2755,6 +2799,8 @@ PyDictKeysObject *mod_keys; _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits; owner = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); @@ -2782,8 +2828,12 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2810,6 +2860,7 @@ PyDictObject *dict; _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); dict = (PyDictObject *)stack_pointer[-1].bits; owner = stack_pointer[-2]; @@ -2832,7 +2883,7 @@ JUMP_TO_JUMP_TARGET(); } } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); stack_pointer += -1; @@ -2865,16 +2916,20 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-2] = attr; - stack_pointer += -1; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR_SLOT: { + case _LOAD_ATTR_SLOT_0: { _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -2894,11 +2949,47 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } + case _LOAD_ATTR_SLOT_1: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_ATTR_SLOT is split on (oparg & 1) */ + case _CHECK_ATTR_CLASS: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2916,24 +3007,50 @@ break; } - case _LOAD_ATTR_CLASS: { + case _LOAD_ATTR_CLASS_0: { _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; } + case _LOAD_ATTR_CLASS_1: { + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + (void)null; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; + PyStackRef_CLOSE(owner); + stack_pointer[-1] = attr; + stack_pointer[0] = null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ + case _LOAD_ATTR_PROPERTY_FRAME: { _PyStackRef owner; _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *fget = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -3939,12 +4056,14 @@ break; } - case _LOAD_METHOD_WITH_VALUES: { + case _LOAD_ATTR_METHOD_WITH_VALUES: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3958,12 +4077,14 @@ break; } - case _LOAD_METHOD_NO_DICT: { + case _LOAD_ATTR_METHOD_NO_DICT: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3980,8 +4101,10 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _PyStackRef owner; _PyStackRef attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -3993,8 +4116,10 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _PyStackRef owner; _PyStackRef attr; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -4018,12 +4143,14 @@ break; } - case _LOAD_METHOD_LAZY_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT: { _PyStackRef owner; _PyStackRef attr; - _PyStackRef self; + _PyStackRef self = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -5517,14 +5644,15 @@ /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ case _MAKE_CALLARGS_A_TUPLE: { - _PyStackRef kwargs_in; + _PyStackRef kwargs_in = PyStackRef_NULL; _PyStackRef callargs; _PyStackRef func; _PyStackRef tuple; - _PyStackRef kwargs_out; - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; + _PyStackRef kwargs_out = PyStackRef_NULL; + oparg = CURRENT_OPARG(); + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -5547,8 +5675,8 @@ PyStackRef_CLOSE(callargs); tuple = PyStackRef_FromPyObjectSteal(tuple_o); } - stack_pointer[-2] = tuple; - stack_pointer[-1] = kwargs_out; + stack_pointer[-1 - (oparg & 1)] = tuple; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; break; } @@ -5625,27 +5753,25 @@ } case _BUILD_SLICE: { - _PyStackRef *args; + _PyStackRef step = PyStackRef_NULL; + _PyStackRef stop; + _PyStackRef start; _PyStackRef slice; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - assert(oparg == 2 || oparg == 3); - _PyStackRef start = args[0]; - _PyStackRef stop = args[1]; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject * step_o = NULL; - if (oparg == 3) { - step_o = PyStackRef_AsPyObjectBorrow(args[2]); - } + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); if (slice_o == NULL) JUMP_TO_ERROR(); slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); break; } @@ -6012,6 +6138,8 @@ case _LOAD_GLOBAL_MODULE: { _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -6022,14 +6150,18 @@ } Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS: { _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -6040,8 +6172,10 @@ } Py_INCREF(res_o); res = PyStackRef_FromPyObjectSteal(res_o); + null = PyStackRef_NULL; stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -6049,6 +6183,8 @@ case _LOAD_ATTR_MODULE: { _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; + oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -6064,8 +6200,12 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 3f8d9db..24561c1 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1836,6 +1836,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) INSTR_SET_OP0(inst, NOP); } break; + case LOAD_GLOBAL: + if (nextop == PUSH_NULL && (oparg & 1) == 0) { + INSTR_SET_OP1(inst, LOAD_GLOBAL, oparg | 1); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); + } + break; case COMPARE_OP: if (nextop == TO_BOOL) { INSTR_SET_OP0(inst, NOP); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7b49f33..4fb3ce6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -833,30 +833,28 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - _PyStackRef *args; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef step = PyStackRef_NULL; _PyStackRef slice; - args = &stack_pointer[-oparg]; - assert(oparg == 2 || oparg == 3); - _PyStackRef start = args[0]; - _PyStackRef stop = args[1]; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); - PyObject * step_o = NULL; - if (oparg == 3) { - step_o = PyStackRef_AsPyObjectBorrow(args[2]); - } + PyObject *step_o = PyStackRef_AsPyObjectBorrow(step); PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } + PyStackRef_CLOSE(start); + PyStackRef_CLOSE(stop); + PyStackRef_XCLOSE(step); if (slice_o == NULL) { - stack_pointer += -oparg; + stack_pointer += -2 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); goto error; } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1708,18 +1706,18 @@ (void)this_instr; _PyStackRef func; _PyStackRef callargs; - _PyStackRef kwargs_in; + _PyStackRef kwargs_in = PyStackRef_NULL; _PyStackRef tuple; - _PyStackRef kwargs_out; + _PyStackRef kwargs_out = PyStackRef_NULL; _PyStackRef func_st; _PyStackRef callargs_st; - _PyStackRef kwargs_st; + _PyStackRef kwargs_st = PyStackRef_NULL; _PyStackRef result; // _MAKE_CALLARGS_A_TUPLE { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; + if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; } + callargs = stack_pointer[-1 - (oparg & 1)]; + func = stack_pointer[-3 - (oparg & 1)]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (PyTuple_CheckExact(callargs_o)) { tuple = callargs; @@ -1761,8 +1759,8 @@ assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, @@ -1805,7 +1803,7 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( @@ -1826,8 +1824,8 @@ assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; + stack_pointer[-1 - (oparg & 1)] = callargs_st; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1837,7 +1835,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE(callargs_st); PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; + if (result_o == NULL) { + stack_pointer += -3 - (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + goto error; + } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC @@ -1845,19 +1847,19 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) goto error; - stack_pointer += 3; + stack_pointer += 2 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-4] = result; - stack_pointer += -3; + stack_pointer[-3 - (oparg & 1)] = result; + stack_pointer += -2 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4846,18 +4848,6 @@ goto PREDICTED_LOAD_SUPER_ATTR; } - TARGET(INSTRUMENTED_LOAD_SUPER_METHOD) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - (void)this_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_METHOD); - /* Skip 1 cache entry */ - // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we - // don't want to specialize instrumented instructions - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PREDICTED_LOAD_SUPER_METHOD; - } - TARGET(INSTRUMENTED_NOT_TAKEN) { _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; @@ -5311,6 +5301,7 @@ (void)this_instr; _PyStackRef owner; _PyStackRef attr; + _PyStackRef self_or_null = PyStackRef_NULL; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; @@ -5318,7 +5309,7 @@ (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -5332,15 +5323,50 @@ /* Skip 8 cache entries */ // _LOAD_ATTR { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o; + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + } + else { + /* Classic, pushes one value. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + /* We need to define self_or_null on all paths */ + self_or_null = PyStackRef_NULL; + } attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5351,6 +5377,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -5368,9 +5395,13 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5381,6 +5412,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { @@ -5404,9 +5436,13 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5422,6 +5458,7 @@ uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); @@ -5434,7 +5471,7 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). @@ -5452,6 +5489,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5483,10 +5521,143 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + /* This object has a __dict__, just not yet created */ + DEOPT_IF(dict != NULL, LOAD_ATTR); + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self = PyStackRef_NULL; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + /* Cached method object */ + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5498,6 +5669,7 @@ _PyStackRef owner; PyDictKeysObject *mod_keys; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE_PUSH_KEYS { @@ -5530,10 +5702,14 @@ attr = PyStackRef_FromPyObjectSteal(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5557,6 +5733,7 @@ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -5601,6 +5778,7 @@ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); PyStackRef_CLOSE(owner); @@ -5634,6 +5812,7 @@ // _LOAD_ATTR_PROPERTY_FRAME { PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -5681,6 +5860,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5704,10 +5884,14 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5719,6 +5903,7 @@ _PyStackRef owner; PyDictObject *dict; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { @@ -5748,7 +5933,7 @@ UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); DEOPT_IF(true, LOAD_ATTR); @@ -5766,10 +5951,14 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); + null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6070,13 +6259,14 @@ _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -6093,13 +6283,15 @@ // _LOAD_GLOBAL { res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) goto error; + null = PyStackRef_NULL; } - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6111,6 +6303,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { @@ -6145,9 +6338,11 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6159,6 +6354,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; _PyStackRef res; + _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION_PUSH_KEYS { @@ -6185,9 +6381,11 @@ res = PyStackRef_FromPyObjectSteal(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); + null = PyStackRef_NULL; } stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6212,198 +6410,6 @@ DISPATCH(); } - TARGET(LOAD_METHOD) { - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD); - PREDICTED_LOAD_METHOD:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - // _SPECIALIZE_LOAD_METHOD - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadMethod(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_METHOD - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_METHOD_LAZY_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_LAZY_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_METHOD); - } - /* Skip 1 cache entry */ - // _LOAD_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_METHOD_NO_DICT) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_NO_DICT); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); - } - /* Skip 2 cache entries */ - // _LOAD_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_METHOD_WITH_VALUES) { - _Py_CODEUNIT* const this_instr = frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_METHOD_WITH_VALUES); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_METHOD); - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_METHOD); - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_METHOD); - } - // _LOAD_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - /* Cached method object */ - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - TARGET(LOAD_NAME) { frame->instr_ptr = next_instr; next_instr += 1; @@ -6483,6 +6489,7 @@ _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; + _PyStackRef null = PyStackRef_NULL; // _SPECIALIZE_LOAD_SUPER_ATTR { class_st = stack_pointer[-2]; @@ -6490,10 +6497,11 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -6507,7 +6515,7 @@ PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( @@ -6527,7 +6535,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6560,9 +6568,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) goto error; attr = PyStackRef_FromPyObjectSteal(attr_o); + null = PyStackRef_NULL; } stack_pointer[0] = attr; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6602,111 +6612,10 @@ DISPATCH(); } - TARGET(LOAD_SUPER_METHOD) { + TARGET(LOAD_SUPER_ATTR_METHOD) { frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PREDICTED_LOAD_SUPER_METHOD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null; - // _SPECIALIZE_LOAD_SUPER_METHOD - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } - } - } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL - { - null = PyStackRef_NULL; - } - stack_pointer[0] = attr; - stack_pointer[1] = null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_SUPER_METHOD_METHOD) { - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_METHOD_METHOD); + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -6721,8 +6630,8 @@ PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_METHOD); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_METHOD); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); + DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -7087,9 +6996,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - _PyStackRef null; - null = PyStackRef_NULL; - stack_pointer[0] = null; + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index d20195d..0e7b481 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -81,8 +81,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL, - [LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_LOAD_SUPER_METHOD] = PY_MONITORING_EVENT_CALL, [RESUME] = -1, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, @@ -128,7 +126,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, [INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, - [INSTRUMENTED_LOAD_SUPER_METHOD] = LOAD_SUPER_METHOD, [INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN, }; @@ -167,8 +164,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER, [LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, - [LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD, - [INSTRUMENTED_LOAD_SUPER_METHOD] = INSTRUMENTED_LOAD_SUPER_METHOD, [NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, [INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2e35ae7..cb6c33f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -3,7 +3,6 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -16,8 +15,8 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_RESERVED, &&TARGET_GET_ITER, + &&TARGET_RESERVED, &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_INTERPRETER_EXIT, @@ -52,6 +51,7 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, @@ -89,12 +89,10 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_DEREF, &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_GLOBAL, - &&TARGET_LOAD_METHOD, &&TARGET_LOAD_NAME, &&TARGET_LOAD_SMALL_INT, &&TARGET_LOAD_SPECIAL, &&TARGET_LOAD_SUPER_ATTR, - &&TARGET_LOAD_SUPER_METHOD, &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, &&TARGET_MATCH_CLASS, @@ -148,6 +146,8 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -198,6 +198,9 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, @@ -208,11 +211,8 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_CONST_MORTAL, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_METHOD_LAZY_DICT, - &&TARGET_LOAD_METHOD_NO_DICT, - &&TARGET_LOAD_METHOD_WITH_VALUES, &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_METHOD_METHOD, + &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_RESUME_CHECK, &&TARGET_SEND_GEN, &&TARGET_STORE_ATTR_INSTANCE_VALUE, @@ -233,11 +233,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, - &&TARGET_INSTRUMENTED_LOAD_SUPER_METHOD, &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, diff --git a/Python/optimizer.c b/Python/optimizer.c index b91caf1..9beb472 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1274,7 +1274,10 @@ uop_optimize( for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; - if (oparg < _PyUop_Replication[opcode]) { + if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { + buffer[pc].opcode = opcode + 1 + (oparg & 1); + } + else if (oparg < _PyUop_Replication[opcode]) { buffer[pc].opcode = opcode + oparg + 1; } else if (is_terminator(&buffer[pc])) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 05f9592..b9ac30e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -109,10 +109,10 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) return NULL; } if (_Py_IsImmortal(res)) { - inst->opcode = _LOAD_CONST_INLINE_BORROW; + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_BORROW_WITH_NULL : _LOAD_CONST_INLINE_BORROW; } else { - inst->opcode = _LOAD_CONST_INLINE; + inst->opcode = (inst->oparg & 1) ? _LOAD_CONST_INLINE_WITH_NULL : _LOAD_CONST_INLINE; } inst->operand0 = (uint64_t)res; return res; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index af834b6..881a607 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -528,8 +528,9 @@ dummy_func(void) { top_out = top_in; } - op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr)) { + op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)offset; (void)owner; } @@ -552,19 +553,15 @@ dummy_func(void) { } } - op(_LOAD_ATTR, (owner -- attr)) { - (void)owner; - attr = sym_new_not_null(ctx); - } - - op(_LOAD_METHOD, (owner -- attr, self_or_null)) { + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); } - op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr)) { + op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr, null if (oparg & 1))) { (void)index; + null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. @@ -592,38 +589,41 @@ dummy_func(void) { (void)owner; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr)) { + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)hint; (void)owner; (void)dict; } - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; } - op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { + op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)descr; (void)owner; } - op(_LOAD_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { + op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { (void)descr; attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { + op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; attr = sym_new_not_null(ctx); self = owner; } - op(_LOAD_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { + op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self if (1))) { (void)descr; attr = sym_new_not_null(ctx); self = owner; @@ -819,7 +819,7 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) { + op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index cb4651a..fa0b4ed 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -928,9 +928,12 @@ case _LOAD_GLOBAL: { JitOptSymbol **res; + JitOptSymbol *null = NULL; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); - stack_pointer += 1; + null = sym_new_null(ctx); + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -963,15 +966,25 @@ case _LOAD_GLOBAL_MODULE_FROM_KEYS: { JitOptSymbol *res; + JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: { JitOptSymbol *res; + JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[-1] = res; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1096,8 +1109,6 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ - /* _INSTRUMENTED_LOAD_SUPER_METHOD is not a viable micro-op for tier 2 */ - case _LOAD_SUPER_ATTR_ATTR: { JitOptSymbol *attr_st; attr_st = sym_new_not_null(ctx); @@ -1107,7 +1118,7 @@ break; } - case _LOAD_SUPER_METHOD_METHOD: { + case _LOAD_SUPER_ATTR_METHOD: { JitOptSymbol *attr; JitOptSymbol *self_or_null; attr = sym_new_not_null(ctx); @@ -1119,31 +1130,21 @@ break; } - case _LOAD_METHOD: { + case _LOAD_ATTR: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self_or_null; + JitOptSymbol *self_or_null = NULL; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); self_or_null = sym_new_unknown(ctx); stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } - case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - owner = stack_pointer[-1]; - (void)owner; - attr = sym_new_not_null(ctx); - stack_pointer[-1] = attr; - break; - } - case _GUARD_TYPE_VERSION: { JitOptSymbol *owner; owner = stack_pointer[-1]; @@ -1181,12 +1182,17 @@ case _LOAD_ATTR_INSTANCE_VALUE: { JitOptSymbol *owner; JitOptSymbol *attr; + JitOptSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)offset; (void)owner; stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1224,9 +1230,11 @@ case _LOAD_ATTR_MODULE_FROM_KEYS: { JitOptSymbol *owner; JitOptSymbol *attr; + JitOptSymbol *null = NULL; owner = stack_pointer[-2]; uint16_t index = (uint16_t)this_instr->operand0; (void)index; + null = sym_new_null(ctx); attr = NULL; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched. @@ -1235,7 +1243,8 @@ assert(PyModule_CheckExact(mod)); PyObject *dict = mod->md_dict; stack_pointer[-2] = attr; - stack_pointer += -1; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); PyObject *res = convert_global_to_const(this_instr, dict); if (res != NULL) { @@ -1245,7 +1254,7 @@ else { this_instr->opcode = _LOAD_ATTR_MODULE; } - stack_pointer += 1; + stack_pointer += 1 - (oparg & 1); assert(WITHIN_STACK_BOUNDS()); } if (attr == NULL) { @@ -1253,7 +1262,8 @@ attr = sym_new_not_null(ctx); } stack_pointer[-2] = attr; - stack_pointer += -1; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -1274,15 +1284,18 @@ JitOptSymbol *dict; JitOptSymbol *owner; JitOptSymbol *attr; + JitOptSymbol *null = NULL; dict = stack_pointer[-1]; owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)hint; (void)owner; (void)dict; stack_pointer[-2] = attr; - stack_pointer += -1; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -1290,12 +1303,17 @@ case _LOAD_ATTR_SLOT: { JitOptSymbol *owner; JitOptSymbol *attr; + JitOptSymbol *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)index; (void)owner; stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1306,12 +1324,17 @@ case _LOAD_ATTR_CLASS: { JitOptSymbol *owner; JitOptSymbol *attr; + JitOptSymbol *null = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); (void)descr; (void)owner; stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1697,10 +1720,10 @@ break; } - case _LOAD_METHOD_WITH_VALUES: { + case _LOAD_ATTR_METHOD_WITH_VALUES: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self; + JitOptSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1713,10 +1736,10 @@ break; } - case _LOAD_METHOD_NO_DICT: { + case _LOAD_ATTR_METHOD_NO_DICT: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self; + JitOptSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -1747,10 +1770,10 @@ break; } - case _LOAD_METHOD_LAZY_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT: { JitOptSymbol *owner; JitOptSymbol *attr; - JitOptSymbol *self; + JitOptSymbol *self = NULL; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; @@ -2239,11 +2262,11 @@ case _MAKE_CALLARGS_A_TUPLE: { JitOptSymbol *tuple; - JitOptSymbol *kwargs_out; + JitOptSymbol *kwargs_out = NULL; tuple = sym_new_not_null(ctx); kwargs_out = sym_new_not_null(ctx); - stack_pointer[-2] = tuple; - stack_pointer[-1] = kwargs_out; + stack_pointer[-1 - (oparg & 1)] = tuple; + if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out; break; } @@ -2292,8 +2315,8 @@ case _BUILD_SLICE: { JitOptSymbol *slice; slice = sym_new_not_null(ctx); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2616,26 +2639,37 @@ case _LOAD_GLOBAL_MODULE: { JitOptSymbol *res; + JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_GLOBAL_BUILTINS: { JitOptSymbol *res; + JitOptSymbol *null = NULL; res = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[0] = res; - stack_pointer += 1; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_MODULE: { JitOptSymbol *attr; + JitOptSymbol *null = NULL; attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index eb59902..fa02234 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -804,7 +804,7 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - uint8_t load_code = load_method ? LOAD_SUPER_METHOD_METHOD : LOAD_SUPER_ATTR_ATTR; + uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; specialize(instr, load_code); return; fail: @@ -1109,7 +1109,7 @@ instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) static int do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, bool shadow, uint32_t shared_keys_version, - DescriptorClassification kind, PyObject *descr, unsigned int tp_version, bool load_method) + DescriptorClassification kind, PyObject *descr, unsigned int tp_version) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); @@ -1117,16 +1117,17 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); return -1; case METHOD: { if (shadow) { goto try_instance; } - if (load_method) { + if (oparg & 1) { if (specialize_attr_loadclassattr(owner, instr, name, descr, tp_version, kind, true, shared_keys_version)) { @@ -1136,7 +1137,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; } } - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } case PROPERTY: @@ -1145,28 +1146,28 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* assert(Py_TYPE(descr) == &PyProperty_Type); PyObject *fget = ((_PyPropertyObject *)descr)->prop_get; if (fget == NULL) { - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); return -1; } if (!Py_IS_TYPE(fget, &PyFunction_Type)) { - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION); return -1; } - if (load_method) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_ATTR_METHOD); + if (!function_check_args(fget, 1, LOAD_ATTR)) { return -1; } - if (!function_check_args(fget, 1, LOAD_ATTR)) { + if (oparg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } /* Don't specialize if PEP 523 is active */ if (_PyInterpreterState_GET()->eval_frame) { - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OTHER); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } #ifdef Py_GIL_DISABLED if (!_PyObject_HasDeferredRefcount(fget)) { - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); return -1; } #endif @@ -1179,10 +1180,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } case OBJECT_SLOT: { - if (load_method) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - return -1; - } PyMemberDescrObject *member = (PyMemberDescrObject *)descr; struct PyMemberDef *dmem = member->d_member; Py_ssize_t offset = dmem->offset; @@ -1207,10 +1204,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } case DUNDER_CLASS: { - if (load_method) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - return -1; - } Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; @@ -1219,20 +1212,16 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return 0; } case OTHER_SLOT: - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT); return -1; case MUTABLE: - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS); return -1; case GETSET_OVERRIDDEN: - SPECIALIZATION_FAIL(load_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { - if (load_method) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - return -1; - } #ifndef Py_GIL_DISABLED // In free-threaded builds it's possible for tp_getattro to change // after the call to analyze_descriptor. That is fine: the version @@ -1244,6 +1233,10 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* if (!function_check_args(descr, 2, LOAD_ATTR)) { return -1; } + if (oparg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + return -1; + } uint32_t version = function_get_version(descr, LOAD_ATTR); if (version == 0) { return -1; @@ -1277,7 +1270,7 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* if (shadow) { goto try_instance; } - if (!load_method) { + if ((oparg & 1) == 0) { if (specialize_attr_loadclassattr(owner, instr, name, descr, tp_version, kind, false, shared_keys_version)) { @@ -1294,10 +1287,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* } Py_UNREACHABLE(); try_instance: - if (load_method) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - return -1; - } if (specialize_dict_access(owner, instr, type, kind, name, tp_version, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { @@ -1307,7 +1296,7 @@ try_instance: } static int -specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, bool load_method) +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) { // 0 is not a valid version uint32_t shared_keys_version = 0; @@ -1316,7 +1305,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na unsigned int tp_version = 0; PyTypeObject *type = Py_TYPE(owner); DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); - int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version, load_method); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); Py_XDECREF(descr); return result; } @@ -1344,40 +1333,7 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_class_load_attr(owner, instr, name); } else { - fail = specialize_instance_load_attr(owner, instr, name, false); - } - - if (fail) { - unspecialize(instr); - } -} - -void -_Py_Specialize_LoadMethod(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) -{ - PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - - assert(ENABLE_SPECIALIZATION_FT); - assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); - PyTypeObject *type = Py_TYPE(owner); - bool fail; - if (!_PyType_IsReady(type)) { - // We *might* not really need this check, but we inherited it from - // PyObject_GenericGetAttr and friends... and this way we still do the - // right thing if someone forgets to call PyType_Ready(type): - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - fail = true; - } - else if (Py_TYPE(owner)->tp_getattro == PyModule_Type.tp_getattro) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - fail = true; - } - else if (PyType_Check(owner)) { - SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OTHER); - fail = true; - } - else { - fail = specialize_instance_load_attr(owner, instr, name, true); + fail = specialize_instance_load_attr(owner, instr, name); } if (fail) { @@ -1619,7 +1575,7 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, #ifdef Py_GIL_DISABLED if (!_PyObject_HasDeferredRefcount(descr)) { - SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); return 0; } #endif @@ -1631,11 +1587,11 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); #endif if (shared_keys_version == 0) { - SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } write_u32(cache->keys_version, shared_keys_version); - specialize(instr, is_method ? LOAD_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; @@ -1645,17 +1601,17 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, else { dictoffset = owner_cls->tp_dictoffset; if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { - SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); return 0; } } if (dictoffset == 0) { - specialize(instr, is_method ? LOAD_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT); } else if (is_method) { PyObject *dict = *(PyObject **) ((char *)owner + dictoffset); if (dict) { - SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } /* Cache entries must be unsigned values, so we offset the @@ -1664,10 +1620,10 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, dictoffset -= MANAGED_DICT_OFFSET; assert(((uint16_t)dictoffset) == dictoffset); cache->dict_offset = (uint16_t)dictoffset; - specialize(instr, LOAD_METHOD_LAZY_DICT); + specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT); } else { - SPECIALIZATION_FAIL(is_method ? LOAD_METHOD : LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); return 0; } } |