diff options
Diffstat (limited to 'Python/specialize.c')
| -rw-r--r-- | Python/specialize.c | 271 |
1 files changed, 164 insertions, 107 deletions
diff --git a/Python/specialize.c b/Python/specialize.c index 6c45320..349ed47 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code); #ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -#endif static uint32_t type_get_version(PyTypeObject *t, int opcode); +#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr) return NON_DESCRIPTOR; } -static DescriptorClassification -analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) +static bool +descriptor_is_class(PyObject *descriptor, PyObject *name) { + return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) && + (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); +} + +#ifndef Py_GIL_DISABLED +static DescriptorClassification +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { bool has_getattr = false; - if (store) { - if (type->tp_setattro != PyObject_GenericSetAttr) { + getattrofunc getattro_slot = type->tp_getattro; + if (getattro_slot == PyObject_GenericGetAttr) { + /* Normal attribute lookup; */ + has_getattr = false; + } + else if (getattro_slot == _Py_slot_tp_getattr_hook || + getattro_slot == _Py_slot_tp_getattro) { + /* One or both of __getattribute__ or __getattr__ may have been + overridden See typeobject.c for why these functions are special. */ + PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool has_custom_getattribute = getattribute != NULL && + getattribute != interp->callable_cache.object__getattribute__; + PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + has_getattr = getattr != NULL; + Py_XDECREF(getattr); + if (has_custom_getattribute) { + if (getattro_slot == _Py_slot_tp_getattro && + !has_getattr && + Py_IS_TYPE(getattribute, &PyFunction_Type)) { + *descr = getattribute; + return GETATTRIBUTE_IS_PYTHON_FUNCTION; + } + /* Potentially both __getattr__ and __getattribute__ are set. + Too complicated */ + Py_DECREF(getattribute); *descr = NULL; return GETSET_OVERRIDDEN; } + /* Potentially has __getattr__ but no custom __getattribute__. + Fall through to usual descriptor analysis. + Usual attribute lookup should only be allowed at runtime + if we can guarantee that there is no way an exception can be + raised. This means some specializations, e.g. specializing + for property() isn't safe. + */ + Py_XDECREF(getattribute); } else { - getattrofunc getattro_slot = type->tp_getattro; - if (getattro_slot == PyObject_GenericGetAttr) { - /* Normal attribute lookup; */ - has_getattr = false; - } - else if (getattro_slot == _Py_slot_tp_getattr_hook || - getattro_slot == _Py_slot_tp_getattro) { - /* One or both of __getattribute__ or __getattr__ may have been - overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - bool has_custom_getattribute = getattribute != NULL && - getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; - if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && - Py_IS_TYPE(getattribute, &PyFunction_Type)) { - *descr = getattribute; - return GETATTRIBUTE_IS_PYTHON_FUNCTION; - } - /* Potentially both __getattr__ and __getattribute__ are set. - Too complicated */ - *descr = NULL; - return GETSET_OVERRIDDEN; - } - /* Potentially has __getattr__ but no custom __getattribute__. - Fall through to usual descriptor analysis. - Usual attribute lookup should only be allowed at runtime - if we can guarantee that there is no way an exception can be - raised. This means some specializations, e.g. specializing - for property() isn't safe. - */ - } - else { - *descr = NULL; - return GETSET_OVERRIDDEN; - } + *descr = NULL; + return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_Lookup(type, name); + PyObject *descriptor = _PyType_LookupRef(type, name); *descr = descriptor; - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { - return DUNDER_CLASS; - } + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } +#endif //!Py_GIL_DISABLED + +static DescriptorClassification +analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) +{ + if (type->tp_setattro != PyObject_GenericSetAttr) { + *descr = NULL; + return GETSET_OVERRIDDEN; + } + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version); + *descr = descriptor; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, false); +} + +static int +specialize_dict_access_inline( + PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int values_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); + assert (index != DKIX_ERROR); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + specialize(instr, values_op); + return 1; +} + +static int +specialize_dict_access_hint( + PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int hint_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + // We found an instance with a __dict__. + if (_PyDict_HasSplitTable(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); + return 0; + } + Py_ssize_t index = _PyDict_LookupIndex(dict, name); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)index; + write_u32(cache->version, tp_version); + specialize(instr, hint_op); + return 1; +} + static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || @@ -956,29 +1027,25 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && - _PyObject_InlineValues(owner)->valid && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) && !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index == DKIX_EMPTY) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); - return 0; + int res; + Py_BEGIN_CRITICAL_SECTION(owner); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + // managed dict, not materialized, inline values valid + res = specialize_dict_access_inline(owner, instr, type, kind, name, + tp_version, base_op, values_op); } - assert(index >= 0); - char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; - Py_ssize_t offset = value_addr - (char *)owner; - if (offset != (uint16_t)offset) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; + else { + // lost race and dict was created, fail specialization + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); + res = 0; } - write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)offset; - specialize(instr, values_op); + Py_END_CRITICAL_SECTION(); + return res; } else { PyDictObject *dict = _PyObject_GetManagedDict(owner); @@ -986,25 +1053,14 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } - // We found an instance with a __dict__. - if (dict->ma_values) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); - return 0; - } - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_DICT : - SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache->index = (uint16_t)index; - write_u32(cache->version, type->tp_version_tag); - specialize(instr, hint_op); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + // materialized managed dict + res = specialize_dict_access_hint(dict, instr, type, kind, name, + tp_version, base_op, hint_op); + Py_END_CRITICAL_SECTION(); + return res; } - return 1; } #ifndef Py_GIL_DISABLED @@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na PyTypeObject *type = Py_TYPE(owner); bool shadow = instance_has_key(owner, name); PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); if (type_get_version(type, LOAD_ATTR) == 0) { return -1; @@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } @@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); + PyObject *descr = NULL; _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (!_PyType_IsReady(type)) { @@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } - PyObject *descr; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); - if (type_get_version(type, STORE_ATTR) == 0) { + unsigned int tp_version = 0; + DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version); + if (tp_version == 0) { goto fail; } + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = STORE_ATTR_SLOT; + write_u32(cache->version, tp_version); + specialize(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_WITH_HINT)) { goto success; } } fail: - STAT_INC(STORE_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + Py_XDECREF(descr); + unspecialize(instr); return; success: - STAT_INC(STORE_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_XDECREF(descr); + return; } #ifndef Py_GIL_DISABLED @@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor(cls, name, &descr, 0); + kind = analyze_descriptor_load(cls, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref if (type_get_version(cls, LOAD_ATTR) == 0) { return -1; } @@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode) } return version; } -#endif // Py_GIL_DISABLED /* Returning 0 indicates a failure. */ static uint32_t @@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode) } return version; } +#endif // Py_GIL_DISABLED void _Py_Specialize_BinarySubscr( |