diff options
35 files changed, 786 insertions, 536 deletions
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 2fb7723..e74fdd4 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -77,12 +77,11 @@ typedef struct _object_stats { uint64_t frees; uint64_t to_freelist; uint64_t from_freelist; - uint64_t new_values; + uint64_t inline_values; uint64_t dict_materialized_on_request; uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; - uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index e004783..6c90c9e 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -79,7 +79,10 @@ typedef struct { typedef struct { uint16_t counter; uint16_t type_version[2]; - uint16_t keys_version[2]; + union { + uint16_t keys_version[2]; + uint16_t dict_offset; + }; uint16_t descr[4]; } _PyLoadMethodCache; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index ef59960..5507bdd 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -11,7 +11,7 @@ extern "C" { #include "pycore_freelist.h" // _PyFreeListState #include "pycore_identifier.h" // _Py_Identifier -#include "pycore_object.h" // PyDictOrValues +#include "pycore_object.h" // PyManagedDictPointer // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); @@ -181,6 +181,10 @@ struct _dictkeysobject { * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order. */ struct _dictvalues { + uint8_t capacity; + uint8_t size; + uint8_t embedded; + uint8_t valid; PyObject *values[1]; }; @@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { size_t index = (size_t)1 << dk->dk_log2_index_bytes; return (&indices[index]); } + static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { assert(dk->dk_kind == DICT_KEYS_GENERAL); return (PyDictKeyEntry*)_DK_ENTRIES(dk); @@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1) #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) -#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1] -#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2] - #ifdef Py_GIL_DISABLED #define DICT_NEXT_VERSION(INTERP) \ (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) @@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK); } -extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj); + PyAPI_FUNC(PyObject *)_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, Py_ssize_t length); +static inline uint8_t * +get_insertion_order_array(PyDictValues *values) +{ + return (uint8_t *)&values->values[values->capacity]; +} + static inline void _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) { assert(ix < SHARED_KEYS_MAX_SIZE); - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; - assert(size+2 < DICT_VALUES_SIZE(values)); - size++; - size_ptr[-size] = (uint8_t)ix; - *size_ptr = size; + int size = values->size; + uint8_t *array = get_insertion_order_array(values); + assert(size < values->capacity); + assert(((uint8_t)ix) == ix); + array[size] = (uint8_t)ix; + values->size = size+1; +} + +static inline size_t +shared_keys_usable_size(PyDictKeysObject *keys) +{ +#ifdef Py_GIL_DISABLED + // dk_usable will decrease for each instance that is created and each + // value that is added. dk_nentries will increase for each value that + // is added. We want to always return the right value or larger. + // We therefore increase dk_nentries first and we decrease dk_usable + // second, and conversely here we read dk_usable first and dk_entries + // second (to avoid the case where we read entries before the increment + // and read usable after the decrement) + return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + + _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); +#else + return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; +#endif } +static inline size_t +_PyInlineValuesSize(PyTypeObject *tp) +{ + PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys; + assert(keys != NULL); + size_t size = shared_keys_usable_size(keys); + size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *)); + assert(prefix_size < 256); + return prefix_size + (size + 1) * sizeof(PyObject *); +} + +int +_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0b17ddf..4fc5e9b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { - Py_INCREF(typeobj); - } + assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); + Py_INCREF(typeobj); _Py_NewReference(op); } @@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_InitializeDict(PyObject *obj); -int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, #endif typedef union { - PyObject *dict; - /* Use a char* to generate a warning if directly assigning a PyDictValues */ - char *values; -} PyDictOrValues; + PyDictObject *dict; +} PyManagedDictPointer; -static inline PyDictOrValues * -_PyObject_DictOrValuesPointer(PyObject *obj) +static inline PyManagedDictPointer * +_PyObject_ManagedDictPointer(PyObject *obj) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET); -} - -static inline int -_PyDictOrValues_IsValues(PyDictOrValues dorv) -{ - return ((uintptr_t)dorv.values) & 1; + return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); } static inline PyDictValues * -_PyDictOrValues_GetValues(PyDictOrValues dorv) -{ - assert(_PyDictOrValues_IsValues(dorv)); - return (PyDictValues *)(dorv.values + 1); -} - -static inline PyObject * -_PyDictOrValues_GetDict(PyDictOrValues dorv) +_PyObject_InlineValues(PyObject *obj) { - assert(!_PyDictOrValues_IsValues(dorv)); - return dorv.dict; -} - -static inline void -_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) -{ - ptr->values = ((char *)values) - 1; + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject)); + return (PyDictValues *)((char *)obj + sizeof(PyObject)); } extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); -extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); // Export for 'math' shared extension diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index de525f7..aa87dc4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, @@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = { [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } }, [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } }, [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index bcb10ab..54dc6dc 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -109,7 +109,7 @@ extern "C" { #define _GUARD_BOTH_INT 347 #define _GUARD_BOTH_UNICODE 348 #define _GUARD_BUILTINS_VERSION 349 -#define _GUARD_DORV_VALUES 350 +#define _GUARD_DORV_NO_DICT 350 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351 #define _GUARD_GLOBALS_VERSION 352 #define _GUARD_IS_FALSE_POP 353 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 51206cd..0f2046f 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG, [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG, @@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_CLASS_0] = 0, [_LOAD_ATTR_CLASS_1] = 0, [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG, - [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = 0, [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", - [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", + [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", @@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_CLASS: return 1; - case _GUARD_DORV_VALUES: + case _GUARD_DORV_NO_DICT: return 1; case _STORE_ATTR_INSTANCE_VALUE: return 2; diff --git a/Include/object.h b/Include/object.h index 9679084..1344332 100644 --- a/Include/object.h +++ b/Include/object.h @@ -629,13 +629,18 @@ given type object has a specified feature. /* Track types initialized using _PyStaticType_InitBuiltin(). */ #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1) +/* The values array is placed inline directly after the rest of + * the object. Implies Py_TPFLAGS_HAVE_GC. + */ +#define Py_TPFLAGS_INLINE_VALUES (1 << 2) + /* Placement of weakref pointers are managed by the VM, not by the type. * The VM will automatically set tp_weaklistoffset. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) /* Placement of dict (and values) pointers are managed by the VM, not by the type. - * The VM will automatically set tp_dictoffset. + * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 04f17a9..1958ecc 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -148,8 +148,8 @@ class PyMemDebugTests(unittest.TestCase): self.assertIn(b'MemoryError', out) *_, count = line.split(b' ') count = int(count) - self.assertLessEqual(count, i*5) - self.assertGreaterEqual(count, i*5-2) + self.assertLessEqual(count, i*10) + self.assertGreaterEqual(count, i*10-4) # Py_GIL_DISABLED requires mimalloc (not malloc) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 0cf0624..4c18141 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -2,7 +2,6 @@ import unittest - testmeths = [ # Binary operations @@ -789,5 +788,81 @@ class ClassTests(unittest.TestCase): self.assertEqual(calls, 100) +from _testinternalcapi import has_inline_values + +Py_TPFLAGS_MANAGED_DICT = (1 << 2) + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class TestInlineValues(unittest.TestCase): + + def test_flags(self): + self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5fb4b81..8829c9a 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1044,20 +1044,13 @@ class TestInstanceDict(unittest.TestCase): c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) + self.assertEqual(c.__dict__, {"a":1, "b": 2}) def test_dict_dematerialization(self): c = C() c.a = 1 c.b = 2 c.__dict__ - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) for _ in range(100): c.a self.assertEqual( @@ -1072,10 +1065,6 @@ class TestInstanceDict(unittest.TestCase): d = c.__dict__ for _ in range(100): c.a - self.assertIs( - _testinternalcapi.get_object_dict_values(c), - None - ) self.assertIs(c.__dict__, d) def test_dict_dematerialization_copy(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst new file mode 100644 index 0000000..5974b18 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst @@ -0,0 +1,4 @@ +The array of values, the ``PyDictValues`` struct is now embedded in the +object during allocation. This provides better performance in the common +case, and does not degrade as much when the object's ``__dict__`` is +materialized. diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 5084bca..cad21bd 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2820,6 +2820,9 @@ static int _testbuffer_exec(PyObject *mod) { Py_SET_TYPE(&NDArray_Type, &PyType_Type); + if (PyType_Ready(&NDArray_Type)) { + return -1; + } if (PyModule_AddType(mod, &NDArray_Type) < 0) { return -1; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e9db6e5..b2af47d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3869,7 +3869,9 @@ PyInit__testcapi(void) return NULL; Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); - + if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { + return NULL; + } if (PyType_Ready(&matmulType) < 0) return NULL; Py_INCREF(&matmulType); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c07652f..d6d50e7 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,7 +15,7 @@ #include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_compile.h" // _PyCompile_CodeGen() #include "pycore_context.h" // _PyContext_NewHamtForTests() -#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_dict.h" // _PyManagedDictPointer_GetValues() #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -1297,14 +1297,13 @@ static PyObject * get_object_dict_values(PyObject *self, PyObject *obj) { PyTypeObject *type = Py_TYPE(obj); - if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { Py_RETURN_NONE; } - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (!_PyDictOrValues_IsValues(dorv)) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (!values->valid) { Py_RETURN_NONE; } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(keys != NULL); int size = (int)keys->dk_nentries; @@ -1784,6 +1783,16 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored)) } #endif +static PyObject * +has_inline_values(PyObject *self, PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1857,6 +1866,7 @@ static PyMethodDef module_functions[] = { _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, + {"has_inline_values", has_inline_values, METH_O}, #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 536746c..58a3d97 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -361,6 +361,10 @@ static int dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value, PyObject **result, int incref_result); +#ifndef NDEBUG +static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); +#endif + #include "clinic/dictobject.c.h" @@ -624,8 +628,9 @@ static inline int get_index_from_order(PyDictObject *mp, Py_ssize_t i) { assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE); - assert(i < (((char *)mp->ma_values)[-2])); - return ((char *)mp->ma_values)[-3-i]; + assert(i < mp->ma_values->size); + uint8_t *array = get_insertion_order_array(mp->ma_values); + return array[i]; } #ifdef DEBUG_PYDICT @@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) else { CHECK(keys->dk_kind == DICT_KEYS_SPLIT); CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE); + if (mp->ma_values->embedded) { + CHECK(mp->ma_values->embedded == 1); + CHECK(mp->ma_values->valid == 1); + } } if (check_content) { @@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) PyMem_Free(keys); } +static size_t +values_size_from_count(size_t count) +{ + assert(count >= 1); + size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *)); + assert(suffix_size < 128); + assert(suffix_size % sizeof(PyObject *) == 0); + return (count + 1) * sizeof(PyObject *) + suffix_size; +} + +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + static inline PyDictValues* new_values(size_t size) { - assert(size >= 1); - size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); - assert(prefix_size < 256); - size_t n = prefix_size + size * sizeof(PyObject *); - uint8_t *mem = PyMem_Malloc(n); - if (mem == NULL) { + size_t n = values_size_from_count(size); + PyDictValues *res = (PyDictValues *)PyMem_Malloc(n); + if (res == NULL) { return NULL; } - assert(prefix_size % sizeof(PyObject *) == 0); - mem[prefix_size-1] = (uint8_t)prefix_size; - return (PyDictValues*)(mem + prefix_size); + res->embedded = 0; + res->size = 0; + assert(size < 256); + res->capacity = (uint8_t)size; + return res; } static inline void free_values(PyDictValues *values, bool use_qsbr) { - int prefix_size = DICT_VALUES_SIZE(values); + assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(((char *)values)-prefix_size); + _PyMem_FreeDelayed(values); return; } #endif - PyMem_Free(((char *)values)-prefix_size); + PyMem_Free(values); } /* Consumes a reference to the keys object */ @@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -static inline size_t -shared_keys_usable_size(PyDictKeysObject *keys) -{ -#ifdef Py_GIL_DISABLED - // dk_usable will decrease for each instance that is created and each - // value that is added. dk_nentries will increase for each value that - // is added. We want to always return the right value or larger. - // We therefore increase dk_nentries first and we decrease dk_usable - // second, and conversely here we read dk_usable first and dk_entries - // second (to avoid the case where we read entries before the increment - // and read usable after the decrement) - return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) + - _Py_atomic_load_ssize_acquire(&keys->dk_nentries)); -#else - return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; -#endif -} - /* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) @@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } - ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb if (values == NULL) goto read_failed; - uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values)); + uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity); if (ix >= (Py_ssize_t)capacity) goto read_failed; @@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op) return; mp = (PyDictObject *) op; + ASSERT_CONSISTENT(mp); numentries = mp->ma_keys->dk_nentries; if (_PyDict_HasSplitTable(mp)) { for (i = 0; i < numentries; i++) { @@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, set_keys(mp, newkeys); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (oldvalues->embedded) { + assert(oldvalues->embedded == 1); + assert(oldvalues->valid == 1); + oldvalues->valid = 0; + } + else { + free_values(oldvalues, IS_DICT_SHARED(mp)); + } } else { // oldkeys is combined. if (oldkeys->dk_kind == DICT_KEYS_GENERAL) { @@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, static void delete_index_from_values(PyDictValues *values, Py_ssize_t ix) { - uint8_t *size_ptr = ((uint8_t *)values)-2; - int size = *size_ptr; + uint8_t *array = get_insertion_order_array(values); + int size = values->size; + assert(size <= values->capacity); int i; - for (i = 1; size_ptr[-i] != ix; i++) { - assert(i <= size); + for (i = 0; array[i] != ix; i++) { + assert(i < size); } - assert(i <= size); + assert(i < size); + size--; for (; i < size; i++) { - size_ptr[-i] = size_ptr[-i-1]; + array[i] = array[i+1]; } - *size_ptr = size -1; + values->size = size; } static int @@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op) mp->ma_version_tag = new_version; /* ...then clear the keys and values */ if (oldvalues != NULL) { - n = oldkeys->dk_nentries; - for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); - free_values(oldvalues, IS_DICT_SHARED(mp)); + if (!oldvalues->embedded) { + n = oldkeys->dk_nentries; + for (i = 0; i < n; i++) + Py_CLEAR(oldvalues->values[i]); + free_values(oldvalues, IS_DICT_SHARED(mp)); + } dictkeys_decref(interp, oldkeys, false); } else { @@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self) PyObject_GC_UnTrack(mp); Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { - for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { - Py_XDECREF(values->values[i]); + if (values->embedded == 0) { + for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + Py_XDECREF(values->values[i]); + } + free_values(values, false); } - free_values(values, false); dictkeys_decref(interp, keys, false); } else if (keys != NULL) { @@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe PyDictKeysObject *okeys = other->ma_keys; // If other is clean, combined, and just allocated, just clone it. - if (other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + if (mp->ma_values == NULL && + other->ma_values == NULL && + other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) + ) { uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); @@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - if (_PyDict_HasSplitTable(mp)) { - free_values(mp->ma_values, IS_DICT_SHARED(mp)); - mp->ma_values = NULL; - } - mp->ma_used = other->ma_used; mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); @@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self) return PyDict_Copy((PyObject *)self); } +/* Copies the values, but does not change the reference + * counts of the objects in the array. */ +static PyDictValues * +copy_values(PyDictValues *values) +{ + PyDictValues *newvalues = new_values(values->capacity); + if (newvalues == NULL) { + PyErr_NoMemory(); + return NULL; + } + newvalues->size = values->size; + uint8_t *values_order = get_insertion_order_array(values); + uint8_t *new_values_order = get_insertion_order_array(newvalues); + memcpy(new_values_order, values_order, values->capacity); + for (int i = 0; i < values->capacity; i++) { + newvalues->values[i] = values->values[i]; + } + assert(newvalues->embedded == 0); + return newvalues; +} + static PyObject * copy_lock_held(PyObject *o) { @@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - size_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues = new_values(size); - if (newvalues == NULL) + PyDictValues *newvalues = copy_values(mp->ma_values); + if (newvalues == NULL) { return PyErr_NoMemory(); + } split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); if (split_copy == NULL) { free_values(newvalues, false); return NULL; } - size_t prefix_size = ((uint8_t *)newvalues)[-1]; - memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1); + for (size_t i = 0; i < newvalues->capacity; i++) { + Py_XINCREF(newvalues->values[i]); + } split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); - for (size_t i = 0; i < size; i++) { - PyObject *value = mp->ma_values->values[i]; - split_copy->ma_values->values[i] = Py_XNewRef(value); - } if (_PyObject_GC_IS_TRACKED(mp)) _PyObject_GC_TRACK(split_copy); return (PyObject *)split_copy; @@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); + if (!mp->ma_values->embedded) { + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); + } } } else { @@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc, return 0; } -static Py_ssize_t -load_values_used_size(PyDictValues *values) -{ - return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values)); -} - static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) @@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, goto concurrent_modification; } - Py_ssize_t used = load_values_used_size(values); + Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size); if (i >= used) { goto fail; } @@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void) return keys; } -#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) - -int +void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); + OBJECT_STAT_INC(inline_values); #ifdef Py_GIL_DISABLED Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable); if (usable > 1) { @@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } #endif size_t size = shared_keys_usable_size(keys); - PyDictValues *values = new_values(size); - if (values == NULL) { - PyErr_NoMemory(); - return -1; - } - assert(((uint8_t *)values)[-1] >= (size + 2)); - ((uint8_t *)values)[-2] = 0; + PyDictValues *values = _PyObject_InlineValues(obj); + assert(size < 256); + values->capacity = (uint8_t)size; + values->size = 0; + values->embedded = 1; + values->valid = 1; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); - return 0; -} - -int -_PyObject_InitializeDict(PyObject *obj) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - if (tp->tp_dictoffset == 0) { - return 0; - } - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - OBJECT_STAT_INC(new_values); - return _PyObject_InitInlineValues(obj, tp); - } - PyObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - } - else { - dict = PyDict_New(); - } - if (dict == NULL) { - return -1; - } - PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - *dictptr = dict; - return 0; + _PyObject_ManagedDictPointer(obj)->dict = NULL; } -static PyObject * +static PyDictObject * make_dict_from_instance_attributes(PyInterpreterState *interp, PyDictKeysObject *keys, PyDictValues *values) { @@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } return res; } -PyObject * -_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) +PyDictObject * +_PyObject_MakeDictFromInstanceAttributes(PyObject *obj) { + PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); return make_dict_from_instance_attributes(interp, keys, values); } -// Return true if the dict was dematerialized, false otherwise. -bool -_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) -{ - assert(_PyObject_DictOrValuesPointer(obj) == dorv); - assert(!_PyDictOrValues_IsValues(*dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); - if (dict == NULL) { - return false; - } - // It's likely that this dict still shares its keys (if it was materialized - // on request and not heavily modified): - if (!PyDict_CheckExact(dict)) { - return false; - } - assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); - if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || - !has_unique_reference((PyObject *)dict)) - { - return false; - } - ensure_shared_on_resize(dict); - assert(dict->ma_values); - // We have an opportunity to do something *really* cool: dematerialize it! - _PyDictKeys_DecRef(dict->ma_keys); - _PyDictOrValues_SetValues(dorv, dict->ma_values); - OBJECT_STAT_INC(dict_dematerialized); - // Don't try this at home, kids: - dict->ma_keys = NULL; - dict->ma_values = NULL; - Py_DECREF(dict); - return true; -} int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, @@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); - assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { Py_hash_t hash = unicode_get_hash(name); @@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, } #endif } + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (ix == DKIX_EMPTY) { - PyObject *dict = make_dict_from_instance_attributes( - interp, keys, values); if (dict == NULL) { - return -1; + dict = make_dict_from_instance_attributes( + interp, keys, values); + if (dict == NULL) { + return -1; + } + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } - _PyObject_DictOrValuesPointer(obj)->dict = dict; if (value == NULL) { - return PyDict_DelItem(dict, name); + return PyDict_DelItem((PyObject *)dict, name); } else { - return PyDict_SetItem(dict, name, value); + return PyDict_SetItem((PyObject *)dict, name, value); } } PyObject *old_value = values->values[ix]; @@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return -1; } _PyDictValues_AddToInsertionOrder(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used++; + } } else { if (value == NULL) { delete_index_from_values(values, ix); + if (dict) { + assert(dict->ma_values == values); + dict->ma_used--; + } } Py_DECREF(old_value); } @@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (_PyManagedDictPointer_IsValues(*managed_dict)) { + PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict); int size = ((uint8_t *)values)[-2]; int count = 0; PyDictKeysObject *keys = CACHED_KEYS(tp); @@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj) CHECK(size == count); } else { - if (dorv_ptr->dict != NULL) { - CHECK(PyDict_Check(dorv_ptr->dict)); + if (managed_dict->dict != NULL) { + CHECK(PyDict_Check(managed_dict->dict)); } } return 1; @@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (tp->tp_dictoffset == 0) { return 1; } - PyObject *dict; - if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictObject *dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { PyDictKeysObject *keys = CACHED_KEYS(tp); for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) { + if (values->values[i] != NULL) { return 0; } } return 1; } - dict = _PyDictOrValues_GetDict(dorv); + dict = _PyObject_ManagedDictPointer(obj)->dict; + } + else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = managed_dict->dict; } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - dict = *dictptr; + dict = (PyDictObject *)*dictptr; } if (dict == NULL) { return 1; @@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) return ((PyDictObject *)dict)->ma_used == 0; } -void -_PyObject_FreeInstanceAttributes(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); - if (!_PyDictOrValues_IsValues(dorv)) { - return; - } - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_XDECREF(values->values[i]); - } - free_values(values, IS_DICT_SHARED((PyDictObject*)self)); -} - int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) { @@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - assert(tp->tp_dictoffset); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(dorv)) { - PyDictValues *values = _PyDictOrValues_GetValues(dorv); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_VISIT(values->values[i]); + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_VISIT(values->values[i]); + } + return 0; } } - else { - PyObject *dict = _PyDictOrValues_GetDict(dorv); - Py_VISIT(dict); - } + Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } void PyObject_ClearManagedDict(PyObject *obj) { + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); PyTypeObject *tp = Py_TYPE(obj); - if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - return; - } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { - Py_CLEAR(values->values[i]); - } - dorv_ptr->dict = NULL; - free_values(values, IS_DICT_SHARED((PyDictObject*)obj)); - } - else { - PyObject *dict = dorv_ptr->dict; + if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; if (dict) { - dorv_ptr->dict = NULL; + _PyDict_DetachFromObject(dict, obj); + _PyObject_ManagedDictPointer(obj)->dict = NULL; Py_DECREF(dict); } + else { + PyDictValues *values = _PyObject_InlineValues(obj); + if (values->valid) { + for (Py_ssize_t i = 0; i < values->capacity; i++) { + Py_CLEAR(values->values[i]); + } + values->valid = 0; + } + } + } + else { + Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict); + } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); +} + +int +_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) +{ + assert(_PyObject_ManagedDictPointer(obj)->dict == mp); + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) { + return 0; + } + assert(mp->ma_values->embedded == 1); + assert(mp->ma_values->valid == 1); + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + Py_BEGIN_CRITICAL_SECTION(mp); + mp->ma_values = copy_values(mp->ma_values); + _PyObject_InlineValues(obj)->valid = 0; + Py_END_CRITICAL_SECTION(); + if (mp->ma_values == NULL) { + return -1; } + assert(_PyObject_InlineValuesConsistencyCheck(obj)); + ASSERT_CONSISTENT(mp); + return 0; } PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - PyObject *dict; PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + PyDictObject *dict = managed_dict->dict; + if (dict == NULL && + (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_InlineValues(obj)->valid + ) { + PyDictValues *values = _PyObject_InlineValues(obj); OBJECT_STAT_INC(dict_materialized_on_request); dict = make_dict_from_instance_attributes( interp, CACHED_KEYS(tp), values); if (dict != NULL) { - dorv_ptr->dict = dict; + managed_dict->dict = (PyDictObject *)dict; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = managed_dict->dict; if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); OBJECT_STAT_INC(dict_materialized_on_request); - dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; + dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); + managed_dict->dict = (PyDictObject *)dict; } } + return Py_XNewRef((PyObject *)dict); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - dict = *dictptr; + PyObject *dict = *dictptr; if (dict == NULL) { PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { @@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) *dictptr = dict = PyDict_New(); } } + return Py_XNewRef(dict); } - return Py_XNewRef(dict); } int @@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, assert(dictptr != NULL); dict = *dictptr; if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) @@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +#ifndef NDEBUG +static int +_PyObject_InlineValuesConsistencyCheck(PyObject *obj) +{ + if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + return 1; + } + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict; + if (dict == NULL) { + return 1; + } + if (dict->ma_values == _PyObject_InlineValues(obj) || + _PyObject_InlineValues(obj)->valid == 0) { + return 1; + } + assert(0); + return 0; +} +#endif diff --git a/Objects/object.c b/Objects/object.c index b4f0fd4..60642d8 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; + managed_dict->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&managed_dict->dict; } PyObject * @@ -1474,21 +1474,19 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + PyDictValues *values = _PyObject_InlineValues(obj); + PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = (PyObject *)managed_dict->dict; } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -1581,29 +1579,27 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + PyDictValues *values = _PyObject_InlineValues(obj); + if (PyUnicode_CheckExact(name)) { + res = _PyObject_GetInstanceAttribute(obj, values, name); + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj); + if (dict == NULL) { + res = NULL; + goto done; + } + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict; } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj); + dict = (PyObject *)managed_dict->dict; + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { @@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; - if (*dictptr == NULL) { - if (_PyObject_InitInlineValues(obj, tp) < 0) { - goto done; - } - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) { + res = _PyObject_StoreInstanceAttribute( + obj, _PyObject_InlineValues(obj), name, value); + goto error_check; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); @@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_ManagedDictPointer(obj)->dict == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } diff --git a/Objects/object_layout.md b/Objects/object_layout.md index 4f379be..3524094 100644 --- a/Objects/object_layout.md +++ b/Objects/object_layout.md @@ -16,7 +16,45 @@ Since the introduction of the cycle GC, there has also been a pre-header. Before 3.11, this pre-header was two words in size. It should be considered opaque to all code except the cycle GC. -## 3.11 pre-header +### 3.13 + +In 3.13, the values array is embedded into the object, so there is no +need for a values pointer (it is just a fixed offset into the object). +So the pre-header is these two fields: + +* weakreflist +* dict_pointer + +If the object has no physical dictionary, then the ``dict_pointer`` +is set to `NULL`. + + +<details> +<summary> 3.12 </summary> + +### 3.12 + +In 3.12, the pointer to the list of weak references is added to the +pre-header. In order to make space for it, the ``dict`` and ``values`` +pointers are combined into a single tagged pointer: + +* weakreflist +* dict_or_values + +If the object has no physical dictionary, then the ``dict_or_values`` +has its low bit set to one, and points to the values array. +If the object has a physical dictionary, then the ``dict_or_values`` +has its low bit set to zero, and points to the dictionary. + +The untagged form is chosen for the dictionary pointer, rather than +the values pointer, to enable the (legacy) C-API function +`_PyObject_GetDictPtr(PyObject *obj)` to work. +</details> + +<details> +<summary> 3.11 </summary> + +### 3.11 In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``. The reason for moving the ``__dict__`` to the pre-header is that it allows @@ -33,27 +71,49 @@ The values pointer refers to the ``PyDictValues`` array which holds the values of the objects's attributes. Should the dictionary be needed, then ``values`` is set to ``NULL`` and the ``dict`` field points to the dictionary. +</details> -## 3.12 pre-header +## Layout of a "normal" Python object -In 3.12, the pointer to the list of weak references is added to the -pre-header. In order to make space for it, the ``dict`` and ``values`` -pointers are combined into a single tagged pointer: +A "normal" Python object is one that doesn't inherit from a builtin +class, doesn't have slots. + +### 3.13 + +In 3.13 the values are embedded into the object, as follows: * weakreflist * dict_or_values +* GC 1 +* GC 2 +* ob_refcnt +* ob_type +* Inlined values: + * Flags + * values 0 + * values 1 + * ... + * Insertion order bytes -If the object has no physical dictionary, then the ``dict_or_values`` -has its low bit set to one, and points to the values array. -If the object has a physical dictionary, then the ``dict_or_values`` -has its low bit set to zero, and points to the dictionary. +This has all the advantages of the layout used in 3.12, plus: +* Access to values is even faster as there is one less load +* Fast access is mostly maintained when the `__dict__` is materialized -The untagged form is chosen for the dictionary pointer, rather than -the values pointer, to enable the (legacy) C-API function -`_PyObject_GetDictPtr(PyObject *obj)` to work. + + +For objects with opaque parts defined by a C extension, +the layout is much the same as for 3.12 + -## Layout of a "normal" Python object in 3.12: + +<details> +<summary> 3.12 </summary> + +### 3.12: + +In 3.12, the header and pre-header form the entire object for "normal" +Python objects: * weakreflist * dict_or_values @@ -62,9 +122,6 @@ the values pointer, to enable the (legacy) C-API function * ob_refcnt * ob_type -For a "normal" Python object, one that doesn't inherit from a builtin -class or have slots, the header and pre-header form the entire object. -  There are several advantages to this layout: @@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension, and `__slots__` looks like this:  +</details> + diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv index c0068d7..731a253 100644 --- a/Objects/object_layout_312.gv +++ b/Objects/object_layout_312.gv @@ -20,6 +20,7 @@ digraph ideal { shape = none label = <<table border="0" cellspacing="0"> <tr><td><b>values</b></td></tr> + <tr><td border="1">Insertion order</td></tr> <tr><td port="0" border="1">values[0]</td></tr> <tr><td border="1">values[1]</td></tr> <tr><td border="1">...</td></tr> diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png Binary files differindex 396dab1..a63d095 100644 --- a/Objects/object_layout_312.png +++ b/Objects/object_layout_312.png diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv new file mode 100644 index 0000000..d853528 --- /dev/null +++ b/Objects/object_layout_313.gv @@ -0,0 +1,45 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>object</b></td></tr> + <tr><td port="w" border="1">weakrefs</td></tr> + <tr><td port="dv" border="1">dict pointer</td></tr> + <tr><td border="1" >GC info 0</td></tr> + <tr><td border="1" >GC info 1</td></tr> + <tr><td port="r" border="1" >refcount</td></tr> + <tr><td port="h" border="1" >__class__</td></tr> + <tr><td border="1">values flags</td></tr> + <tr><td port="0" border="1">values[0]</td></tr> + <tr><td border="1">values[1]</td></tr> + <tr><td border="1">...</td></tr> + <tr><td border="1">Insertion order</td></tr> + </table>> + + ] + + class [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>class</b></td></tr> + <tr><td port="head" bgcolor="lightgreen" border="1">...</td></tr> + <tr><td border="1" bgcolor="lightgreen">dict_offset</td></tr> + <tr><td border="1" bgcolor="lightgreen">...</td></tr> + <tr><td port="k" border="1" bgcolor="lightgreen">cached_keys</td></tr> + </table>> + ] + + keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"] + NULL [ label = " NULL"; shape="plain"] + object:w -> NULL + object:h -> class:head + object:dv -> NULL + class:k -> keys + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png Binary files differnew file mode 100644 index 0000000..f7059c2 --- /dev/null +++ b/Objects/object_layout_313.png diff --git a/Objects/object_layout_full_313.gv b/Objects/object_layout_full_313.gv new file mode 100644 index 0000000..551312d --- /dev/null +++ b/Objects/object_layout_full_313.gv @@ -0,0 +1,25 @@ +digraph ideal { + + rankdir = "LR" + + + object [ + shape = none + label = <<table border="0" cellspacing="0"> + <tr><td><b>object</b></td></tr> + <tr><td port="w" border="1">weakrefs</td></tr> + <tr><td port="dv" border="1">dict pointer</td></tr> + <tr><td border="1" >GC info 0</td></tr> + <tr><td border="1" >GC info 1</td></tr> + <tr><td port="r" border="1" >refcount</td></tr> + <tr><td port="h" border="1" >__class__</td></tr> + <tr><td border="1">opaque (extension) data </td></tr> + <tr><td border="1">...</td></tr> + <tr><td border="1">__slot__ 0</td></tr> + <tr><td border="1">...</td></tr> + </table>> + ] + + oop [ label = "pointer"; shape="plain"] + oop -> object:r +} diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png Binary files differnew file mode 100644 index 0000000..7352ec6 --- /dev/null +++ b/Objects/object_layout_full_313.png diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2ef79fb..6df9986 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _PyType_NewManagedObject(PyTypeObject *type) { - assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES); assert(_PyType_IS_GC(type)); assert(type->tp_new == PyBaseObject_Type.tp_new); assert(type->tp_alloc == PyType_GenericAlloc); @@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type) if (obj == NULL) { return PyErr_NoMemory(); } - _PyObject_DictOrValuesPointer(obj)->dict = NULL; - if (_PyObject_InitInlineValues(obj, type)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) * flag to indicate when that is safe) it does not seem worth the memory * savings. An example type that doesn't need the +1 is a subclass of * tuple. See GH-100659 and GH-81381. */ - const size_t size = _PyObject_VAR_SIZE(type, nitems+1); + size_t size = _PyObject_VAR_SIZE(type, nitems+1); const size_t presize = _PyType_PreHeaderSize(type); + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + assert(type->tp_itemsize == 0); + size += _PyInlineValuesSize(type); + } char *alloc = _PyObject_MallocWithType(type, size + presize); if (alloc == NULL) { return PyErr_NoMemory(); @@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) else { _PyObject_InitVar((PyVarObject *)obj, type, nitems); } + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + _PyObject_InitInlineValues(obj, type); + } return obj; } @@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self) if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { PyObject_ClearManagedDict(self); } + else { + assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) == + (type->tp_flags & Py_TPFLAGS_INLINE_VALUES)); + } } else if (type->tp_dictoffset != base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self) /* If we added a dict, DECREF it, or free inline values. */ if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - _PyObject_FreeInstanceAttributes(self); - } - else { - Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr)); - } - dorv_ptr->values = NULL; + PyObject_ClearManagedDict(self); } else if (type->tp_dictoffset && !base->tp_dictoffset) { PyObject **dictptr = _PyObject_ComputedDictPointer(self); @@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) return func(descr, obj, value); } /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } if (value != NULL && !PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, " "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - Py_XSETREF(*dictptr, Py_XNewRef(value)); + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + PyObject_ClearManagedDict(obj); + _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value); + } + else { + dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_CLEAR(*dictptr); + *dictptr = Py_XNewRef(value); + } return 0; } @@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (obj == NULL) { return NULL; } - if (_PyObject_InitializeDict(obj)) { - Py_DECREF(obj); - return NULL; - } return obj; } @@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } + if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != + ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) + { + goto differs; + } /* The above does not check for the preheader */ if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) @@ -6137,14 +6144,18 @@ object_set_class(PyObject *self, PyObject *value, void *closure) if (compatible_for_assignment(oldto, newto, "__class__")) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ - assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER)); - _PyObject_GetDictPtr(self); - if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self))) - { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - return -1; + if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict; + if (dict == NULL) { + dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self); + if (dict == NULL) { + return -1; + } + _PyObject_ManagedDictPointer(self)->dict = dict; + } + if (_PyDict_DetachFromObject(dict, self)) { + return -1; + } } if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_INCREF(newto); @@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type) return -1; } } + if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) { + type->tp_flags |= Py_TPFLAGS_INLINE_VALUES; + } return 0; } @@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type) /* Historically, all static types were immutable. See bpo-43908 */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + /* Static types must be immortal */ + _Py_SetImmortalUntracked((PyObject *)type); } int res; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bfb378c..ce208aa 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1897,14 +1897,12 @@ dummy_func( op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid); } split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1947,16 +1945,15 @@ dummy_func( op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); } op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -2070,16 +2067,17 @@ dummy_func( DISPATCH_INLINED(new_frame); } - op(_GUARD_DORV_VALUES, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); + op(_GUARD_DORV_NO_DICT, (owner -- owner)) { + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict); + DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -2094,7 +2092,7 @@ dummy_func( macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + _GUARD_TYPE_VERSION + - _GUARD_DORV_VALUES + + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { @@ -2102,9 +2100,8 @@ dummy_func( assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv)); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2898,9 +2895,8 @@ dummy_func( } op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { @@ -2972,10 +2968,9 @@ dummy_func( unused/2 + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } @@ -2993,7 +2988,7 @@ dummy_func( unused/1 + _GUARD_TYPE_VERSION + _CHECK_ATTR_METHOD_LAZY_DICT + - unused/2 + + unused/1 + _LOAD_ATTR_METHOD_LAZY_DICT; inst(INSTRUMENTED_CALL, (unused/3 -- )) { @@ -3294,6 +3289,7 @@ dummy_func( DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ce0dc23..82f2171 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1753,9 +1753,8 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); break; } @@ -1766,8 +1765,7 @@ (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1784,8 +1782,7 @@ (void)null; owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -1837,9 +1834,8 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (dict == NULL) JUMP_TO_JUMP_TARGET(); assert(PyDict_CheckExact((PyObject *)dict)); break; @@ -1852,8 +1848,8 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; uint16_t hint = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -1967,12 +1963,13 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _GUARD_DORV_VALUES: { + case _GUARD_DORV_NO_DICT: { PyObject *owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET(); + if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET(); break; } @@ -1982,9 +1979,9 @@ owner = stack_pointer[-1]; value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND(); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -2568,9 +2565,8 @@ case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { PyObject *owner; owner = stack_pointer[-1]; - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); break; } @@ -2658,9 +2654,9 @@ case _CHECK_ATTR_METHOD_LAZY_DICT: { PyObject *owner; owner = stack_pointer[-1]; - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND(); + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ if (dict != NULL) JUMP_TO_JUMP_TARGET(); break; diff --git a/Python/gc.c b/Python/gc.c index a37c1b1..a487388 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2031,11 +2031,16 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize) return op; } + PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 4524382..7e4137a 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1639,7 +1639,11 @@ PyObject * _PyObject_GC_New(PyTypeObject *tp) { size_t presize = _PyType_PreHeaderSize(tp); - PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize); + size_t size = _PyObject_SIZE(tp); + if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) { + size += _PyInlineValuesSize(tp); + } + PyObject *op = gc_alloc(tp, size, presize); if (op == NULL) { return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e8e2397..6ee794a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -870,6 +870,7 @@ DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; @@ -3680,15 +3681,13 @@ // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - attr = _PyDictOrValues_GetValues(dorv)->values[index]; + attr = _PyObject_InlineValues(owner)->values[index]; DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); @@ -3721,13 +3720,13 @@ } // _CHECK_ATTR_METHOD_LAZY_DICT { - Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset; - assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)owner + dictoffset); + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } - /* Skip 2 cache entries */ + /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3798,9 +3797,8 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -3914,9 +3912,8 @@ } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR); } // _GUARD_KEYS_VERSION { @@ -4026,17 +4023,16 @@ // _CHECK_ATTR_WITH_HINT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL, LOAD_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { @@ -5315,19 +5311,20 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } - // _GUARD_DORV_VALUES + // _GUARD_DORV_NO_DICT { - assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + assert(Py_TYPE(owner)->tp_dictoffset < 0); + assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR); + DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { uint16_t index = read_u16(&this_instr[4].cache); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); - PyDictValues *values = _PyDictOrValues_GetValues(dorv); + assert(_PyObject_ManagedDictPointer(owner)->dict == NULL); + PyDictValues *values = _PyObject_InlineValues(owner); PyObject *old_value = values->values[index]; values->values[index] = value; if (old_value == NULL) { @@ -5389,9 +5386,8 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; DEOPT_IF(dict == NULL, STORE_ATTR); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index df73cc0..b4a1da8 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1104,7 +1104,7 @@ /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ - case _GUARD_DORV_VALUES: { + case _GUARD_DORV_NO_DICT: { break; } diff --git a/Python/specialize.c b/Python/specialize.c index c1edf88..f1e32d0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -188,7 +188,7 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs); fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs); @@ -197,7 +197,6 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -479,12 +478,11 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20 - #define SPEC_FAIL_ATTR_SHADOWED 21 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24 -#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25 + #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28 @@ -558,6 +556,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 #define SPEC_FAIL_CALL_METACLASS 31 +#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -829,11 +828,7 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(*dorv) || - _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - // Virtual dictionary + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) { PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); @@ -850,7 +845,8 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); + PyDictObject *dict = managed_dict->dict; if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; @@ -1258,15 +1254,8 @@ PyObject *descr, DescriptorClassification kind, bool is_method) assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) - { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); - return 0; - } Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != DKIX_EMPTY) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); @@ -1282,10 +1271,16 @@ PyObject *descr, DescriptorClassification kind, bool is_method) instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; } else { - Py_ssize_t dictoffset = owner_cls->tp_dictoffset; - if (dictoffset < 0 || dictoffset > INT16_MAX) { - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); - return 0; + Py_ssize_t dictoffset; + if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + dictoffset = MANAGED_DICT_OFFSET; + } + else { + dictoffset = owner_cls->tp_dictoffset; + if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } } if (dictoffset == 0) { instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT; @@ -1296,8 +1291,12 @@ PyObject *descr, DescriptorClassification kind, bool is_method) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - assert(owner_cls->tp_dictoffset > 0); - assert(owner_cls->tp_dictoffset <= INT16_MAX); + /* Cache entries must be unsigned values, so we offset the + * dictoffset by MANAGED_DICT_OFFSET. + * We do the reverese offset in LOAD_ATTR_METHOD_LAZY_DICT */ + dictoffset -= MANAGED_DICT_OFFSET; + assert(((uint16_t)dictoffset) == dictoffset); + cache->dict_offset = (uint16_t)dictoffset; instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT; } else { @@ -1729,8 +1728,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp) SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); return NULL; } if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index ddafcf9..4261378 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -359,11 +359,10 @@ def has_error_without_pop(op: parser.InstDef) -> bool: NON_ESCAPING_FUNCTIONS = ( "Py_INCREF", - "_PyDictOrValues_IsValues", - "_PyObject_DictOrValuesPointer", - "_PyDictOrValues_GetValues", + "_PyManagedDictPointer_IsValues", + "_PyObject_ManagedDictPointer", + "_PyObject_InlineValues", "_PyDictValues_AddToInsertionOrder", - "_PyObject_MakeInstanceAttributesFromDict", "Py_DECREF", "_Py_DECREF_SPECIALIZED", "DECREF_INPUTS_AND_REUSE_FLOAT", diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 656667a..74165ac 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -66,10 +66,12 @@ def _type_unsigned_short_ptr(): def _type_unsigned_int_ptr(): return gdb.lookup_type('unsigned int').pointer() - def _sizeof_void_p(): return gdb.lookup_type('void').pointer().sizeof +def _sizeof_pyobject(): + return gdb.lookup_type('PyObject').sizeof + def _managed_dict_offset(): # See pycore_object.h pyobj = gdb.lookup_type("PyObject") @@ -79,6 +81,7 @@ def _managed_dict_offset(): return -3 * _sizeof_void_p() +Py_TPFLAGS_INLINE_VALUES = (1 << 2) Py_TPFLAGS_MANAGED_DICT = (1 << 4) Py_TPFLAGS_HEAPTYPE = (1 << 9) Py_TPFLAGS_LONG_SUBCLASS = (1 << 24) @@ -493,11 +496,12 @@ class HeapTypeObjectPtr(PyObjectPtr): has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT if not has_values: return None - ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset() - char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference() - if (int(char_ptr) & 1) == 0: + obj_ptr = self._gdbval.cast(_type_char_ptr()) + dict_ptr_ptr = obj_ptr + _managed_dict_offset() + dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference() + if int(dict_ptr): return None - char_ptr += 1 + char_ptr = obj_ptr + _sizeof_pyobject() values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer()) values = values_ptr['values'] return PyKeysValuesPair(self.get_cached_keys(), values) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 8dc590b..f7ed98f 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -394,7 +394,7 @@ class Stats: return result def get_object_stats(self) -> dict[str, tuple[int, int]]: - total_materializations = self._data.get("Object new values", 0) + total_materializations = self._data.get("Object inline values", 0) total_allocations = self._data.get("Object allocations", 0) + self._data.get( "Object allocations from freelist", 0 ) @@ -1094,8 +1094,7 @@ def object_stats_section() -> Section: Below, "allocations" means "allocations that are not from a freelist". Total allocations = "Allocations from freelist" + "Allocations". - "New values" is the number of values arrays created for objects with - managed dicts. + "Inline values" is the number of values arrays inlined into objects. The cache hit/miss numbers are for the MRO cache, split into dunder and other names. |
