diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/bytecodes.c | 65 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 93 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 86 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 4 | ||||
| -rw-r--r-- | Python/specialize.c | 271 |
5 files changed, 342 insertions, 177 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cf089c3..30c12dd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1467,7 +1467,7 @@ dummy_func( }; specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -1476,7 +1476,7 @@ dummy_func( } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_STORE_ATTR, (v, owner --)) { @@ -2129,7 +2129,18 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - EXIT_IF(tp->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version); + } + + op(_GUARD_TYPE_VERSION_AND_LOCK, (type_version/2, owner -- owner)) { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + EXIT_IF(!LOCK_OBJECT(owner_o)); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { @@ -2336,8 +2347,11 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - EXIT_IF(_PyObject_GetManagedDict(owner_o)); - EXIT_IF(_PyObject_InlineValues(owner_o)->valid == 0); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + EXIT_IF(true); + } } op(_STORE_ATTR_INSTANCE_VALUE, (offset/1, value, owner --)) { @@ -2347,21 +2361,20 @@ dummy_func( assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } macro(STORE_ATTR_INSTANCE_VALUE) = unused/1 + - _GUARD_TYPE_VERSION + + _GUARD_TYPE_VERSION_AND_LOCK + _GUARD_DORV_NO_DICT + _STORE_ATTR_INSTANCE_VALUE; @@ -2370,16 +2383,34 @@ dummy_func( assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); + DEOPT_IF(!LOCK_OBJECT(dict)); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true); + } _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -2395,10 +2426,12 @@ dummy_func( op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o)); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9bfc3e7..6e752c5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2574,13 +2574,34 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - if (tp->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { _PyStackRef owner; owner = stack_pointer[-1]; @@ -2910,13 +2931,13 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (_PyObject_InlineValues(owner_o)->valid == 0) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -2932,15 +2953,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -2961,30 +2981,50 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries) { + if (!LOCK_OBJECT(dict)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #endif + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *old_value = ep->me_value; if (old_value == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -3002,10 +3042,15 @@ value = stack_pointer[-2]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); stack_pointer += -2; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ac89891..ee5c55a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5319,7 +5319,7 @@ uint32_t type_version = read_u32(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { @@ -5388,7 +5388,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { @@ -5433,7 +5433,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_METHOD_LAZY_DICT { @@ -5476,7 +5476,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT @@ -5512,7 +5512,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5611,7 +5611,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT @@ -5642,7 +5642,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { @@ -5688,7 +5688,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME @@ -5750,7 +5750,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { @@ -5787,7 +5787,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, LOAD_ATTR); } // _CHECK_ATTR_WITH_HINT { @@ -7314,7 +7314,7 @@ owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; @@ -7325,7 +7325,7 @@ } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 3 cache entries */ // _STORE_ATTR @@ -7353,21 +7353,29 @@ _PyStackRef owner; _PyStackRef value; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION + // _GUARD_TYPE_VERSION_AND_LOCK { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _GUARD_DORV_NO_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(_PyObject_GetManagedDict(owner_o), STORE_ATTR); - DEOPT_IF(_PyObject_InlineValues(owner_o)->valid == 0, STORE_ATTR); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + DEOPT_IF(true, STORE_ATTR); + } } // _STORE_ATTR_INSTANCE_VALUE { @@ -7378,15 +7386,14 @@ assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; - *value_ptr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } - else { - Py_DECREF(old_value); - } + UNLOCK_OBJECT(owner_o); + Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } stack_pointer += -2; @@ -7408,17 +7415,19 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + DEOPT_IF(!LOCK_OBJECT(owner_o), STORE_ATTR); char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; - *(PyObject **)addr = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); Py_XDECREF(old_value); PyStackRef_CLOSE(owner); } @@ -7441,7 +7450,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, STORE_ATTR); } // _STORE_ATTR_WITH_HINT { @@ -7451,18 +7460,35 @@ assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(!LOCK_OBJECT(dict), STORE_ATTR); + #ifdef Py_GIL_DISABLED + if (dict != _PyObject_GetManagedDict(owner_o)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } + #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), STORE_ATTR); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } PyObject *old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, STORE_ATTR); + } _PyFrame_SetStackPointer(frame, stack_pointer); _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - ep->me_value = PyStackRef_AsPyObjectSteal(value); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); // old_value should be DECREFed after GC track checking is done, if not, it could raise a segmentation fault, // when dict only holds the strong reference to value in ep->me_value. Py_XDECREF(old_value); @@ -7815,7 +7841,7 @@ uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version, TO_BOOL); } // _REPLACE_WITH_TRUE { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2c3133d..b46079e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1121,6 +1121,10 @@ break; } + case _GUARD_TYPE_VERSION_AND_LOCK: { + break; + } + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { break; } diff --git a/Python/specialize.c b/Python/specialize.c index 6c45320..349ed47 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code); #ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -#endif static uint32_t type_get_version(PyTypeObject *t, int opcode); +#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr) return NON_DESCRIPTOR; } -static DescriptorClassification -analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store) +static bool +descriptor_is_class(PyObject *descriptor, PyObject *name) { + return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) && + (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); +} + +#ifndef Py_GIL_DISABLED +static DescriptorClassification +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { bool has_getattr = false; - if (store) { - if (type->tp_setattro != PyObject_GenericSetAttr) { + getattrofunc getattro_slot = type->tp_getattro; + if (getattro_slot == PyObject_GenericGetAttr) { + /* Normal attribute lookup; */ + has_getattr = false; + } + else if (getattro_slot == _Py_slot_tp_getattr_hook || + getattro_slot == _Py_slot_tp_getattro) { + /* One or both of __getattribute__ or __getattr__ may have been + overridden See typeobject.c for why these functions are special. */ + PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool has_custom_getattribute = getattribute != NULL && + getattribute != interp->callable_cache.object__getattribute__; + PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + has_getattr = getattr != NULL; + Py_XDECREF(getattr); + if (has_custom_getattribute) { + if (getattro_slot == _Py_slot_tp_getattro && + !has_getattr && + Py_IS_TYPE(getattribute, &PyFunction_Type)) { + *descr = getattribute; + return GETATTRIBUTE_IS_PYTHON_FUNCTION; + } + /* Potentially both __getattr__ and __getattribute__ are set. + Too complicated */ + Py_DECREF(getattribute); *descr = NULL; return GETSET_OVERRIDDEN; } + /* Potentially has __getattr__ but no custom __getattribute__. + Fall through to usual descriptor analysis. + Usual attribute lookup should only be allowed at runtime + if we can guarantee that there is no way an exception can be + raised. This means some specializations, e.g. specializing + for property() isn't safe. + */ + Py_XDECREF(getattribute); } else { - getattrofunc getattro_slot = type->tp_getattro; - if (getattro_slot == PyObject_GenericGetAttr) { - /* Normal attribute lookup; */ - has_getattr = false; - } - else if (getattro_slot == _Py_slot_tp_getattr_hook || - getattro_slot == _Py_slot_tp_getattro) { - /* One or both of __getattribute__ or __getattr__ may have been - overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); - PyInterpreterState *interp = _PyInterpreterState_GET(); - bool has_custom_getattribute = getattribute != NULL && - getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; - if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && - Py_IS_TYPE(getattribute, &PyFunction_Type)) { - *descr = getattribute; - return GETATTRIBUTE_IS_PYTHON_FUNCTION; - } - /* Potentially both __getattr__ and __getattribute__ are set. - Too complicated */ - *descr = NULL; - return GETSET_OVERRIDDEN; - } - /* Potentially has __getattr__ but no custom __getattribute__. - Fall through to usual descriptor analysis. - Usual attribute lookup should only be allowed at runtime - if we can guarantee that there is no way an exception can be - raised. This means some specializations, e.g. specializing - for property() isn't safe. - */ - } - else { - *descr = NULL; - return GETSET_OVERRIDDEN; - } + *descr = NULL; + return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_Lookup(type, name); + PyObject *descriptor = _PyType_LookupRef(type, name); *descr = descriptor; - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { - return DUNDER_CLASS; - } + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } +#endif //!Py_GIL_DISABLED + +static DescriptorClassification +analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) +{ + if (type->tp_setattro != PyObject_GenericSetAttr) { + *descr = NULL; + return GETSET_OVERRIDDEN; + } + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version); + *descr = descriptor; + if (descriptor_is_class(descriptor, name)) { + return DUNDER_CLASS; + } + return classify_descriptor(descriptor, false); +} + +static int +specialize_dict_access_inline( + PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int values_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(PyUnicode_CheckExact(name)); + Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); + assert (index != DKIX_ERROR); + if (index == DKIX_EMPTY) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); + return 0; + } + assert(index >= 0); + char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; + Py_ssize_t offset = value_addr - (char *)owner; + if (offset != (uint16_t)offset) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)offset; + write_u32(cache->version, tp_version); + specialize(instr, values_op); + return 1; +} + +static int +specialize_dict_access_hint( + PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, + int base_op, int hint_op) +{ + _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + // We found an instance with a __dict__. + if (_PyDict_HasSplitTable(dict)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); + return 0; + } + Py_ssize_t index = _PyDict_LookupIndex(dict, name); + if (index != (uint16_t)index) { + SPECIALIZATION_FAIL(base_op, + index == DKIX_EMPTY ? + SPEC_FAIL_ATTR_NOT_IN_DICT : + SPEC_FAIL_OUT_OF_RANGE); + return 0; + } + cache->index = (uint16_t)index; + write_u32(cache->version, tp_version); + specialize(instr, hint_op); + return 1; +} + static int specialize_dict_access( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, + DescriptorClassification kind, PyObject *name, unsigned int tp_version, int base_op, int values_op, int hint_op) { assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT || @@ -956,29 +1027,25 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; } - _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && - _PyObject_InlineValues(owner)->valid && + FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) && !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; - assert(PyUnicode_CheckExact(name)); - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); - assert (index != DKIX_ERROR); - if (index == DKIX_EMPTY) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS); - return 0; + int res; + Py_BEGIN_CRITICAL_SECTION(owner); + PyDictObject *dict = _PyObject_GetManagedDict(owner); + if (dict == NULL) { + // managed dict, not materialized, inline values valid + res = specialize_dict_access_inline(owner, instr, type, kind, name, + tp_version, base_op, values_op); } - assert(index >= 0); - char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; - Py_ssize_t offset = value_addr - (char *)owner; - if (offset != (uint16_t)offset) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE); - return 0; + else { + // lost race and dict was created, fail specialization + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER); + res = 0; } - write_u32(cache->version, type->tp_version_tag); - cache->index = (uint16_t)offset; - specialize(instr, values_op); + Py_END_CRITICAL_SECTION(); + return res; } else { PyDictObject *dict = _PyObject_GetManagedDict(owner); @@ -986,25 +1053,14 @@ specialize_dict_access( SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } - // We found an instance with a __dict__. - if (dict->ma_values) { - SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); - return 0; - } - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); - if (index != (uint16_t)index) { - SPECIALIZATION_FAIL(base_op, - index == DKIX_EMPTY ? - SPEC_FAIL_ATTR_NOT_IN_DICT : - SPEC_FAIL_OUT_OF_RANGE); - return 0; - } - cache->index = (uint16_t)index; - write_u32(cache->version, type->tp_version_tag); - specialize(instr, hint_op); + int res; + Py_BEGIN_CRITICAL_SECTION(dict); + // materialized managed dict + res = specialize_dict_access_hint(dict, instr, type, kind, name, + tp_version, base_op, hint_op); + Py_END_CRITICAL_SECTION(); + return res; } - return 1; } #ifndef Py_GIL_DISABLED @@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na PyTypeObject *type = Py_TYPE(owner); bool shadow = instance_has_key(owner, name); PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); if (type_get_version(type, LOAD_ATTR) == 0) { return -1; @@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR, - LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) + if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } @@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); + PyObject *descr = NULL; _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); if (!_PyType_IsReady(type)) { @@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } - PyObject *descr; - DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); - if (type_get_version(type, STORE_ATTR) == 0) { + unsigned int tp_version = 0; + DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version); + if (tp_version == 0) { goto fail; } + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); - instr->op.code = STORE_ATTR_SLOT; + write_u32(cache->version, tp_version); + specialize(instr, STORE_ATTR_SLOT); goto success; } case DUNDER_CLASS: @@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE); goto fail; case ABSENT: - if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR, - STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT)) - { + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, + STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, + STORE_ATTR_WITH_HINT)) { goto success; } } fail: - STAT_INC(STORE_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = STORE_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + Py_XDECREF(descr); + unspecialize(instr); return; success: - STAT_INC(STORE_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + Py_XDECREF(descr); + return; } #ifndef Py_GIL_DISABLED @@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor(cls, name, &descr, 0); + kind = analyze_descriptor_load(cls, name, &descr); + Py_XDECREF(descr); // turn strong ref into a borrowed ref if (type_get_version(cls, LOAD_ATTR) == 0) { return -1; } @@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode) } return version; } -#endif // Py_GIL_DISABLED /* Returning 0 indicates a failure. */ static uint32_t @@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode) } return version; } +#endif // Py_GIL_DISABLED void _Py_Specialize_BinarySubscr( |