diff options
Diffstat (limited to 'Python/specialize.c')
| -rw-r--r-- | Python/specialize.c | 120 |
1 files changed, 56 insertions, 64 deletions
diff --git a/Python/specialize.c b/Python/specialize.c index 4a94aaf..66dce8c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -58,12 +58,9 @@ static uint8_t adaptive_opcodes[256] = { /* The number of cache entries required for a "family" of instructions. */ static uint8_t cache_requirements[256] = { - [LOAD_ATTR] = 1, // _PyAdaptiveEntry - [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [STORE_SUBSCR] = 0, [CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ [PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ - [STORE_ATTR] = 1, // _PyAdaptiveEntry }; Py_ssize_t _Py_QuickenedCount = 0; @@ -641,11 +638,10 @@ initial_counter_value(void) { static int -specialize_module_load_attr( - PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, - _PyAdaptiveEntry *cache0, int opcode, - int opcode_module) +specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, int opcode, int opcode_module) { + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyModuleObject *m = (PyModuleObject *)owner; PyObject *value = NULL; assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); @@ -676,8 +672,8 @@ specialize_module_load_attr( SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } - cache0->version = keys_version; - cache0->index = (uint16_t)index; + write_u32(cache->version, keys_version); + cache->index = (uint16_t)index; *instr = _Py_MAKECODEUNIT(opcode_module, _Py_OPARG(*instr)); return 0; } @@ -765,7 +761,6 @@ static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, DescriptorClassification kind, PyObject *name, - _PyAdaptiveEntry *cache0, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || @@ -775,6 +770,7 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyObject **dictptr = _PyObject_ManagedDictPointer(owner); PyDictObject *dict = (PyDictObject *)*dictptr; if (dict == NULL) { @@ -787,8 +783,8 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } - cache0->version = type->tp_version_tag; - cache0->index = (uint16_t)index; + write_u32(cache->version, type->tp_version_tag); + cache->index = (uint16_t)index; *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr)); } else { @@ -804,20 +800,22 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); return 0; } - cache0->index = (uint16_t)hint; - cache0->version = type->tp_version_tag; + cache->index = (uint16_t)hint; + write_u32(cache->version, type->tp_version_tag); *instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr)); } return 1; } int -_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache) +_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - _PyAdaptiveEntry *cache0 = &cache->adaptive; + assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] == + INLINE_CACHE_ENTRIES_LOAD_ATTR); + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (PyModule_CheckExact(owner)) { - int err = specialize_module_load_attr(owner, instr, name, cache0, - LOAD_ATTR, LOAD_ATTR_MODULE); + int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR, + LOAD_ATTR_MODULE); if (err) { goto fail; } @@ -856,8 +854,8 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp } assert(dmem->type == T_OBJECT_EX); assert(offset > 0); - cache0->index = (uint16_t)offset; - cache0->version = type->tp_version_tag; + cache->index = (uint16_t)offset; + write_u32(cache->version, type->tp_version_tag); *instr = _Py_MAKECODEUNIT(LOAD_ATTR_SLOT, _Py_OPARG(*instr)); goto success; } @@ -865,8 +863,8 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp { Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); - cache0->index = (uint16_t)offset; - cache0->version = type->tp_version_tag; + cache->index = (uint16_t)offset; + write_u32(cache->version, type->tp_version_tag); *instr = _Py_MAKECODEUNIT(LOAD_ATTR_SLOT, _Py_OPARG(*instr)); goto success; } @@ -887,41 +885,33 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp break; } int err = specialize_dict_access( - owner, instr, type, kind, name, cache0, + owner, instr, type, kind, name, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT ); if (err < 0) { return -1; } if (err) { - if (_Py_OPCODE(instr[0]) == LOAD_ATTR_INSTANCE_VALUE) { - // Note: instr[-1] exists because there's something on the stack, - // and instr[-2] exists because there's at least a RESUME as well. - if (_Py_OPCODE(instr[-1]) == LOAD_FAST) { - instr[-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE, _Py_OPARG(instr[-1])); - if (_Py_OPCODE(instr[-2]) == LOAD_FAST__LOAD_FAST) { - instr[-2] = _Py_MAKECODEUNIT(LOAD_FAST, _Py_OPARG(instr[-2])); - } - } - } goto success; } fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - cache_backoff(cache0); + cache->counter = ADAPTIVE_CACHE_BACKOFF; return 0; success: STAT_INC(LOAD_ATTR, success); assert(!PyErr_Occurred()); - cache0->counter = initial_counter_value(); + cache->counter = initial_counter_value(); return 0; } int -_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache) +_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - _PyAdaptiveEntry *cache0 = &cache->adaptive; + assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] == + INLINE_CACHE_ENTRIES_STORE_ATTR); + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (PyModule_CheckExact(owner)) { SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); @@ -954,8 +944,8 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S } assert(dmem->type == T_OBJECT_EX); assert(offset > 0); - cache0->index = (uint16_t)offset; - cache0->version = type->tp_version_tag; + cache->index = (uint16_t)offset; + write_u32(cache->version, type->tp_version_tag); *instr = _Py_MAKECODEUNIT(STORE_ATTR_SLOT, _Py_OPARG(*instr)); goto success; } @@ -978,7 +968,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S } int err = specialize_dict_access( - owner, instr, type, kind, name, cache0, + owner, instr, type, kind, name, STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT ); if (err < 0) { @@ -990,12 +980,12 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - cache_backoff(cache0); + cache->counter = ADAPTIVE_CACHE_BACKOFF; return 0; success: STAT_INC(STORE_ATTR, success); assert(!PyErr_Occurred()); - cache0->counter = initial_counter_value(); + cache->counter = initial_counter_value(); return 0; } @@ -1037,18 +1027,18 @@ load_method_fail_kind(DescriptorClassification kind) #endif static int -specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, - _PyAttrCache *cache1, _PyObjectCache *cache2) +specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name) { - + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyObject *descr = NULL; DescriptorClassification kind = 0; kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0); switch (kind) { case METHOD: case NON_DESCRIPTOR: - cache1->tp_version = ((PyTypeObject *)owner)->tp_version_tag; - cache2->obj = descr; + write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag); + write_obj(cache->descr, descr); *instr = _Py_MAKECODEUNIT(LOAD_METHOD_CLASS, _Py_OPARG(*instr)); return 0; #ifdef Py_STATS @@ -1078,16 +1068,18 @@ typedef enum { // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. int -_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache) +_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { - _PyAdaptiveEntry *cache0 = &cache->adaptive; - _PyAttrCache *cache1 = &cache[-1].attr; - _PyObjectCache *cache2 = &cache[-2].obj; + assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] == + INLINE_CACHE_ENTRIES_LOAD_METHOD); + _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); if (PyModule_CheckExact(owner)) { - int err = specialize_module_load_attr(owner, instr, name, cache0, - LOAD_METHOD, LOAD_METHOD_MODULE); + assert(INLINE_CACHE_ENTRIES_LOAD_ATTR <= + INLINE_CACHE_ENTRIES_LOAD_METHOD); + int err = specialize_module_load_attr(owner, instr, name, LOAD_METHOD, + LOAD_METHOD_MODULE); if (err) { goto fail; } @@ -1099,7 +1091,7 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, } } if (PyType_Check(owner)) { - int err = specialize_class_load_method(owner, instr, name, cache1, cache2); + int err = specialize_class_load_method(owner, instr, name); if (err) { goto fail; } @@ -1157,7 +1149,7 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - cache1->dk_version = keys_version; + write_u32(cache->keys_version, keys_version); } switch(dictkind) { case NO_DICT: @@ -1167,12 +1159,12 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, *instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_VALUES, _Py_OPARG(*instr)); break; case MANAGED_DICT: - *(int16_t *)&cache0->index = (int16_t)MANAGED_DICT_OFFSET; + *(int16_t *)&cache->dict_offset = (int16_t)MANAGED_DICT_OFFSET; *instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr)); break; case OFFSET_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - cache0->index = (uint16_t)owner_cls->tp_dictoffset; + cache->dict_offset = (uint16_t)owner_cls->tp_dictoffset; *instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr)); break; } @@ -1190,18 +1182,18 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ - cache1->tp_version = owner_cls->tp_version_tag; - cache2->obj = descr; + write_u32(cache->type_version, owner_cls->tp_version_tag); + write_obj(cache->descr, descr); // Fall through. success: STAT_INC(LOAD_METHOD, success); assert(!PyErr_Occurred()); - cache0->counter = initial_counter_value(); + cache->counter = initial_counter_value(); return 0; fail: STAT_INC(LOAD_METHOD, failure); assert(!PyErr_Occurred()); - cache_backoff(cache0); + cache->counter = ADAPTIVE_CACHE_BACKOFF; return 0; } @@ -1238,7 +1230,7 @@ _Py_Specialize_LoadGlobal( goto fail; } cache->index = (uint16_t)index; - write32(&cache->module_keys_version, keys_version); + write_u32(cache->module_keys_version, keys_version); *instr = _Py_MAKECODEUNIT(LOAD_GLOBAL_MODULE, _Py_OPARG(*instr)); goto success; } @@ -1273,7 +1265,7 @@ _Py_Specialize_LoadGlobal( goto fail; } cache->index = (uint16_t)index; - write32(&cache->module_keys_version, globals_version); + write_u32(cache->module_keys_version, globals_version); cache->builtin_keys_version = (uint16_t)builtins_version; *instr = _Py_MAKECODEUNIT(LOAD_GLOBAL_BUILTIN, _Py_OPARG(*instr)); goto success; @@ -1393,7 +1385,7 @@ _Py_Specialize_BinarySubscr( goto fail; } assert(cls->tp_version_tag != 0); - write32(&cache->type_version, cls->tp_version_tag); + write_u32(cache->type_version, cls->tp_version_tag); int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0 || version != (uint16_t)version) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS); |