diff options
Diffstat (limited to 'Include')
| -rw-r--r-- | Include/internal/pycore_interp.h | 4 | ||||
| -rw-r--r-- | Include/internal/pycore_object.h | 10 | ||||
| -rw-r--r-- | Include/internal/pycore_tstate.h | 8 | ||||
| -rw-r--r-- | Include/internal/pycore_typeid.h | 75 | ||||
| -rw-r--r-- | Include/internal/pycore_uniqueid.h | 72 |
5 files changed, 83 insertions, 86 deletions
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a1898d9..ade69be 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -35,7 +35,7 @@ extern "C" { #include "pycore_qsbr.h" // struct _qsbr_state #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeid.h" // struct _Py_type_id_pool +#include "pycore_uniqueid.h" // struct _Py_unique_id_pool #include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state @@ -221,7 +221,7 @@ struct _is { #if defined(Py_GIL_DISABLED) struct _mimalloc_interp_state mimalloc; struct _brc_state brc; // biased reference counting state - struct _Py_type_id_pool type_ids; + struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS]; #endif diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 80b5888..0af13b1 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,7 +14,7 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_typeid.h" // _PyType_IncrefSlow +#include "pycore_uniqueid.h" // _PyType_IncrefSlow #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1) @@ -335,12 +335,12 @@ _Py_INCREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); # endif _Py_INCREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]++; + tstate->refcounts.values[ht->unique_id]++; } else { // The slow path resizes the thread-local refcount array if necessary. @@ -368,12 +368,12 @@ _Py_DECREF_TYPE(PyTypeObject *type) // Unsigned comparison so that `unique_id=-1`, which indicates that // per-thread refcounting has been disabled on this type, is handled by // the "else". - if ((size_t)ht->unique_id < (size_t)tstate->types.size) { + if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) { # ifdef Py_REF_DEBUG _Py_DECREF_DecRefTotal(); # endif _Py_DECREF_STAT_INC(); - tstate->types.refcounts[ht->unique_id]--; + tstate->refcounts.values[ht->unique_id]--; } else { // Directly decref the type if the type id is not assigned or if diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index f681b64..a72ef44 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl { struct _Py_freelists freelists; struct _brc_thread_state brc; struct { - // The thread-local refcounts for heap type objects - Py_ssize_t *refcounts; + // The per-thread refcounts + Py_ssize_t *values; // Size of the refcounts array. Py_ssize_t size; - // If set, don't use thread-local refcounts + // If set, don't use per-thread refcounts int is_finalized; - } types; + } refcounts; #endif #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) diff --git a/Include/internal/pycore_typeid.h b/Include/internal/pycore_typeid.h deleted file mode 100644 index e64d144..0000000 --- a/Include/internal/pycore_typeid.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef Py_INTERNAL_TYPEID_H -#define Py_INTERNAL_TYPEID_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#ifdef Py_GIL_DISABLED - -// This contains code for allocating unique ids to heap type objects -// and re-using those ids when the type is deallocated. -// -// The type ids are used to implement per-thread reference counts of -// heap type objects to avoid contention on the reference count fields -// of heap type objects. Static type objects are immortal, so contention -// is not an issue for those types. -// -// Type id of -1 is used to indicate a type doesn't use thread-local -// refcounting. This value is used when a type object is finalized by the GC -// and during interpreter shutdown to allow the type object to be -// deallocated promptly when the object's refcount reaches zero. -// -// Each entry implicitly represents a type id based on it's offset in the -// table. Non-allocated entries form a free-list via the 'next' pointer. -// Allocated entries store the corresponding PyTypeObject. -typedef union _Py_type_id_entry { - // Points to the next free type id, when part of the freelist - union _Py_type_id_entry *next; - - // Stores the type object when the id is assigned - PyHeapTypeObject *type; -} _Py_type_id_entry; - -struct _Py_type_id_pool { - PyMutex mutex; - - // combined table of types with allocated type ids and unallocated - // type ids. - _Py_type_id_entry *table; - - // Next entry to allocate inside 'table' or NULL - _Py_type_id_entry *freelist; - - // size of 'table' - Py_ssize_t size; -}; - -// Assigns the next id from the pool of type ids. -extern void _PyType_AssignId(PyHeapTypeObject *type); - -// Releases the allocated type id back to the pool. -extern void _PyType_ReleaseId(PyHeapTypeObject *type); - -// Merges the thread-local reference counts into the corresponding types. -extern void _PyType_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Like _PyType_MergeThreadLocalRefcounts, but also frees the thread-local -// array of refcounts. -extern void _PyType_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate); - -// Frees the interpreter's pool of type ids. -extern void _PyType_FinalizeIdPool(PyInterpreterState *interp); - -// Increfs the type, resizing the thread-local refcount array if necessary. -PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); - -#endif /* Py_GIL_DISABLED */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_TYPEID_H */ diff --git a/Include/internal/pycore_uniqueid.h b/Include/internal/pycore_uniqueid.h new file mode 100644 index 0000000..8f3b441 --- /dev/null +++ b/Include/internal/pycore_uniqueid.h @@ -0,0 +1,72 @@ +#ifndef Py_INTERNAL_UNIQUEID_H +#define Py_INTERNAL_UNIQUEID_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_GIL_DISABLED + +// This contains code for allocating unique ids to objects for per-thread +// reference counting. +// +// Per-thread reference counting is used along with deferred reference +// counting to avoid scaling bottlenecks due to reference count contention. +// +// An id of -1 is used to indicate that an object doesn't use per-thread +// refcounting. This value is used when the object is finalized by the GC +// and during interpreter shutdown to allow the object to be +// deallocated promptly when the object's refcount reaches zero. +// +// Each entry implicitly represents a unique id based on its offset in the +// table. Non-allocated entries form a free-list via the 'next' pointer. +// Allocated entries store the corresponding PyObject. +typedef union _Py_unique_id_entry { + // Points to the next free type id, when part of the freelist + union _Py_unique_id_entry *next; + + // Stores the object when the id is assigned + PyObject *obj; +} _Py_unique_id_entry; + +struct _Py_unique_id_pool { + PyMutex mutex; + + // combined table of object with allocated unique ids and unallocated ids. + _Py_unique_id_entry *table; + + // Next entry to allocate inside 'table' or NULL + _Py_unique_id_entry *freelist; + + // size of 'table' + Py_ssize_t size; +}; + +// Assigns the next id from the pool of ids. +extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj); + +// Releases the allocated id back to the pool. +extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id); + +// Merges the per-thread reference counts into the corresponding objects. +extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread +// array of refcounts. +extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate); + +// Frees the interpreter's pool of type ids. +extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp); + +// Increfs the type, resizing the per-thread refcount array if necessary. +PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type); + +#endif /* Py_GIL_DISABLED */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNIQUEID_H */ |