bpo-45947: Place dict and values pointer at fixed (negative) offset just before GC header. (GH-29879)

* Place __dict__ immediately before GC header for plain Python objects.

* Fix up lazy dict creation logic to use managed dict pointers.

* Manage values pointer, placing them directly before managed dict pointers.

* Convert hint-based load/store attr specialization target managed dict classes.

* Specialize LOAD_METHOD for managed dict objects.

* Remove unsafe _PyObject_GC_Calloc function.

* Remove unsafe _PyObject_GC_Malloc() function.

* Add comment explaning use of Py_TPFLAGS_MANAGED_DICT.

3 files changed, 46 insertions, 79 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index a8bbad3..446772d 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3599,9 +3599,9 @@ check_eval_breaker:
             _PyAttrCache *cache1 = &caches[-1].attr;
             assert(cache1->tp_version != 0);
             DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR);
-            assert(tp->tp_dictoffset > 0);
-            assert(tp->tp_inline_values_offset > 0);
-            PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset);
+            assert(tp->tp_dictoffset < 0);
+            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictValues *values = *_PyObject_ValuesPointer(owner);
             DEOPT_IF(values == NULL, LOAD_ATTR);
             res = values->values[cache0->index];
             DEOPT_IF(res == NULL, LOAD_ATTR);
@@ -3633,8 +3633,8 @@ check_eval_breaker:
             _PyAttrCache *cache1 = &caches[-1].attr;
             assert(cache1->tp_version != 0);
             DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR);
-            assert(tp->tp_dictoffset > 0);
-            PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset);
+            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
             DEOPT_IF(dict == NULL, LOAD_ATTR);
             assert(PyDict_CheckExact((PyObject *)dict));
             PyObject *name = GETITEM(names, cache0->original_oparg);
@@ -3701,9 +3701,8 @@ check_eval_breaker:
             _PyAttrCache *cache1 = &caches[-1].attr;
             assert(cache1->tp_version != 0);
             DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR);
-            assert(tp->tp_dictoffset > 0);
-            assert(tp->tp_inline_values_offset > 0);
-            PyDictValues *values = *(PyDictValues **)(((char *)owner) + tp->tp_inline_values_offset);
+            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictValues *values = *_PyObject_ValuesPointer(owner);
             DEOPT_IF(values == NULL, STORE_ATTR);
             STAT_INC(STORE_ATTR, hit);
             int index = cache0->index;
@@ -3731,8 +3730,8 @@ check_eval_breaker:
             _PyAttrCache *cache1 = &caches[-1].attr;
             assert(cache1->tp_version != 0);
             DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR);
-            assert(tp->tp_dictoffset > 0);
-            PyDictObject *dict = *(PyDictObject **)(((char *)owner) + tp->tp_dictoffset);
+            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
             DEOPT_IF(dict == NULL, STORE_ATTR);
             assert(PyDict_CheckExact((PyObject *)dict));
             PyObject *name = GETITEM(names, cache0->original_oparg);
@@ -4506,9 +4505,8 @@ check_eval_breaker:
             _PyObjectCache *cache2 = &caches[-2].obj;
 
             DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD);
-            assert(self_cls->tp_dictoffset > 0);
-            assert(self_cls->tp_inline_values_offset > 0);
-            PyDictObject *dict = *(PyDictObject **)(((char *)self) + self_cls->tp_dictoffset);
+            assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self);
             DEOPT_IF(dict != NULL, LOAD_METHOD);
             DEOPT_IF(((PyHeapTypeObject *)self_cls)->ht_cached_keys->dk_version != cache1->dk_version_or_hint, LOAD_METHOD);
             STAT_INC(LOAD_METHOD, hit);
diff --git a/Python/specialize.c b/Python/specialize.c
index b384675..cdc5353 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -451,6 +451,7 @@ initial_counter_value(void) {
 #define SPEC_FAIL_NON_OBJECT_SLOT 14
 #define SPEC_FAIL_READ_ONLY 15
 #define SPEC_FAIL_AUDITED_SLOT 16
+#define SPEC_FAIL_NOT_MANAGED_DICT 17
 
 /* Methods */
 
@@ -506,7 +507,7 @@ specialize_module_load_attr(
     PyObject *value = NULL;
     PyObject *getattr;
     _Py_IDENTIFIER(__getattr__);
-    assert(owner->ob_type->tp_inline_values_offset == 0);
+    assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
     PyDictObject *dict = (PyDictObject *)m->md_dict;
     if (dict == NULL) {
         SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT);
@@ -634,66 +635,44 @@ specialize_dict_access(
     assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
         kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);
     // No descriptor, or non overriding.
-    if (type->tp_dictoffset < 0) {
-        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
+    if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NOT_MANAGED_DICT);
         return 0;
     }
-    if (type->tp_dictoffset > 0) {
-        PyObject **dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
-        PyDictObject *dict = (PyDictObject *)*dictptr;
-        if (type->tp_inline_values_offset && dict == NULL) {
-            // Virtual dictionary
-            PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
-            assert(type->tp_inline_values_offset > 0);
-            assert(PyUnicode_CheckExact(name));
-            Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
-            assert (index != DKIX_ERROR);
-            if (index != (uint16_t)index) {
-                SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
-                return 0;
-            }
-            cache1->tp_version = type->tp_version_tag;
-            cache0->index = (uint16_t)index;
-            *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr));
+    PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
+    PyDictObject *dict = (PyDictObject *)*dictptr;
+    if (dict == NULL) {
+        // Virtual dictionary
+        PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
+        assert(PyUnicode_CheckExact(name));
+        Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
+        assert (index != DKIX_ERROR);
+        if (index != (uint16_t)index) {
+            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
             return 0;
         }
-        else {
-            if (dict == NULL || !PyDict_CheckExact(dict)) {
-                SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
-                return 0;
-            }
-            // We found an instance with a __dict__.
-            PyObject *value = NULL;
-            Py_ssize_t hint =
-                _PyDict_GetItemHint(dict, name, -1, &value);
-            if (hint != (uint32_t)hint) {
-                SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
-                return 0;
-            }
-            cache1->dk_version_or_hint = (uint32_t)hint;
-            cache1->tp_version = type->tp_version_tag;
-            *instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr));
-            return 1;
-        }
+        cache1->tp_version = type->tp_version_tag;
+        cache0->index = (uint16_t)index;
+        *instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr));
     }
-    assert(type->tp_dictoffset == 0);
-    /* No attribute in instance dictionary */
-    switch(kind) {
-        case NON_OVERRIDING:
-        case BUILTIN_CLASSMETHOD:
-        case PYTHON_CLASSMETHOD:
-            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NON_OVERRIDING_DESCRIPTOR);
-            return 0;
-        case NON_DESCRIPTOR:
-            /* To do -- Optimize this case */
-            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NOT_DESCRIPTOR);
+    else {
+        if (!PyDict_CheckExact(dict)) {
+            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
             return 0;
-        case ABSENT:
-            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_EXPECTED_ERROR);
+        }
+        // We found an instance with a __dict__.
+        PyObject *value = NULL;
+        Py_ssize_t hint =
+            _PyDict_GetItemHint(dict, name, -1, &value);
+        if (hint != (uint32_t)hint) {
+            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
             return 0;
-        default:
-            Py_UNREACHABLE();
+        }
+        cache1->dk_version_or_hint = (uint32_t)hint;
+        cache1->tp_version = type->tp_version_tag;
+        *instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr));
     }
+    return 1;
 }
 
 int
@@ -965,12 +944,6 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
         }
         goto success;
     }
-    // Technically this is fine for bound method calls, but it's uncommon and
-    // slightly slower at runtime to get dict.
-    if (owner_cls->tp_dictoffset < 0) {
-        SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE);
-        goto fail;
-    }
 
     PyObject *descr = NULL;
     DesciptorClassification kind = 0;
@@ -980,9 +953,8 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
         SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
         goto fail;
     }
-    if (owner_cls->tp_inline_values_offset) {
-        PyObject **owner_dictptr = _PyObject_DictPointer(owner);
-        assert(owner_dictptr);
+    if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        PyObject **owner_dictptr = _PyObject_ManagedDictPointer(owner);
         if (*owner_dictptr) {
             SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_IS_ATTR);
             goto fail;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 13fae79..af4f926 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1681,10 +1681,7 @@ _PySys_GetSizeOf(PyObject *o)
         return (size_t)-1;
     }
 
-    /* add gc_head size */
-    if (_PyObject_IS_GC(o))
-        return ((size_t)size) + sizeof(PyGC_Head);
-    return (size_t)size;
+    return (size_t)size + _PyType_PreHeaderSize(Py_TYPE(o));
 }
 
 static PyObject *