gh-112529: Implement GC for free-threaded builds (#114262)

* gh-112529: Implement GC for free-threaded builds

This implements a mark and sweep GC for the free-threaded builds of
CPython. The implementation relies on mimalloc to find GC tracked
objects (i.e., "containers").

5 files changed, 131 insertions, 10 deletions

diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h
index 4ab93ee..dfb1283 100644
--- a/Include/internal/pycore_freelist.h
+++ b/Include/internal/pycore_freelist.h
@@ -20,6 +20,7 @@ extern "C" {
 #  define PyFloat_MAXFREELIST 100
 #  define PyContext_MAXFREELIST 255
 # define _PyAsyncGen_MAXFREELIST 80
+#  define _PyObjectStackChunk_MAXFREELIST 4
 #else
 #  define PyTuple_NFREELISTS 0
 #  define PyTuple_MAXFREELIST 0
@@ -27,6 +28,7 @@ extern "C" {
 #  define PyFloat_MAXFREELIST 0
 #  define PyContext_MAXFREELIST 0
 #  define _PyAsyncGen_MAXFREELIST 0
+#  define _PyObjectStackChunk_MAXFREELIST 0
 #endif
 
 struct _Py_list_state {
@@ -93,6 +95,13 @@ struct _Py_async_gen_state {
 #endif
 };
 
+struct _PyObjectStackChunk;
+
+struct _Py_object_stack_state {
+    struct _PyObjectStackChunk *free_list;
+    Py_ssize_t numfree;
+};
+
 typedef struct _Py_freelist_state {
     struct _Py_float_state float_state;
     struct _Py_tuple_state tuple_state;
@@ -100,6 +109,7 @@ typedef struct _Py_freelist_state {
     struct _Py_slice_state slice_state;
     struct _Py_context_state context_state;
     struct _Py_async_gen_state async_gen_state;
+    struct _Py_object_stack_state object_stack_state;
 } _PyFreeListState;
 
 #ifdef __cplusplus
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index d53de97..b362a29 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -37,10 +37,22 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
 }
 
 
+/* Bit flags for ob_gc_bits (in Py_GIL_DISABLED builds) */
+#ifdef Py_GIL_DISABLED
+#  define _PyGC_BITS_TRACKED        (1)
+#  define _PyGC_BITS_FINALIZED      (2)
+#  define _PyGC_BITS_UNREACHABLE    (4)
+#  define _PyGC_BITS_FROZEN         (8)
+#endif
+
 /* True if the object is currently tracked by the GC. */
 static inline int _PyObject_GC_IS_TRACKED(PyObject *op) {
+#ifdef Py_GIL_DISABLED
+    return (op->ob_gc_bits & _PyGC_BITS_TRACKED) != 0;
+#else
     PyGC_Head *gc = _Py_AS_GC(op);
     return (gc->_gc_next != 0);
+#endif
 }
 #define _PyObject_GC_IS_TRACKED(op) _PyObject_GC_IS_TRACKED(_Py_CAST(PyObject*, op))
 
@@ -107,24 +119,29 @@ static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) {
     gc->_gc_prev = ((gc->_gc_prev & ~_PyGC_PREV_MASK) | uprev);
 }
 
-static inline int _PyGCHead_FINALIZED(PyGC_Head *gc) {
-    return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0);
-}
-static inline void _PyGCHead_SET_FINALIZED(PyGC_Head *gc) {
-    gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED;
-}
-
 static inline int _PyGC_FINALIZED(PyObject *op) {
+#ifdef Py_GIL_DISABLED
+    return (op->ob_gc_bits & _PyGC_BITS_FINALIZED) != 0;
+#else
     PyGC_Head *gc = _Py_AS_GC(op);
-    return _PyGCHead_FINALIZED(gc);
+    return ((gc->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0);
+#endif
 }
 static inline void _PyGC_SET_FINALIZED(PyObject *op) {
+#ifdef Py_GIL_DISABLED
+    op->ob_gc_bits |= _PyGC_BITS_FINALIZED;
+#else
     PyGC_Head *gc = _Py_AS_GC(op);
-    _PyGCHead_SET_FINALIZED(gc);
+    gc->_gc_prev |= _PyGC_PREV_MASK_FINALIZED;
+#endif
 }
 static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) {
+#ifdef Py_GIL_DISABLED
+    op->ob_gc_bits &= ~_PyGC_BITS_FINALIZED;
+#else
     PyGC_Head *gc = _Py_AS_GC(op);
     gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED;
+#endif
 }
 
 
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index ec14c07..4e52ffc 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -322,6 +322,9 @@ static inline void _PyObject_GC_TRACK(
                           "object is in generation which is garbage collected",
                           filename, lineno, __func__);
 
+#ifdef Py_GIL_DISABLED
+    op->ob_gc_bits |= _PyGC_BITS_TRACKED;
+#else
     PyInterpreterState *interp = _PyInterpreterState_GET();
     PyGC_Head *generation0 = interp->gc.generation0;
     PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev);
@@ -329,6 +332,7 @@ static inline void _PyObject_GC_TRACK(
     _PyGCHead_SET_PREV(gc, last);
     _PyGCHead_SET_NEXT(gc, generation0);
     generation0->_gc_prev = (uintptr_t)gc;
+#endif
 }
 
 /* Tell the GC to stop tracking this object.
@@ -352,6 +356,9 @@ static inline void _PyObject_GC_UNTRACK(
                           "object not tracked by the garbage collector",
                           filename, lineno, __func__);
 
+#ifdef Py_GIL_DISABLED
+    op->ob_gc_bits &= ~_PyGC_BITS_TRACKED;
+#else
     PyGC_Head *gc = _Py_AS_GC(op);
     PyGC_Head *prev = _PyGCHead_PREV(gc);
     PyGC_Head *next = _PyGCHead_NEXT(gc);
@@ -359,6 +366,7 @@ static inline void _PyObject_GC_UNTRACK(
     _PyGCHead_SET_PREV(next, prev);
     gc->_gc_next = 0;
     gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED;
+#endif
 }
 
 // Macros to accept any type for the parameter, and to automatically pass
diff --git a/Include/internal/pycore_object_stack.h b/Include/internal/pycore_object_stack.h
new file mode 100644
index 0000000..1dc1c15
--- /dev/null
+++ b/Include/internal/pycore_object_stack.h
@@ -0,0 +1,84 @@
+#ifndef Py_INTERNAL_OBJECT_STACK_H
+#define Py_INTERNAL_OBJECT_STACK_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+// _PyObjectStack is a stack of Python objects implemented as a linked list of
+// fixed size buffers.
+
+// Chosen so that _PyObjectStackChunk is a power-of-two size.
+#define _Py_OBJECT_STACK_CHUNK_SIZE 254
+
+typedef struct _PyObjectStackChunk {
+    struct _PyObjectStackChunk *prev;
+    Py_ssize_t n;
+    PyObject *objs[_Py_OBJECT_STACK_CHUNK_SIZE];
+} _PyObjectStackChunk;
+
+typedef struct _PyObjectStack {
+    _PyObjectStackChunk *head;
+} _PyObjectStack;
+
+
+extern _PyObjectStackChunk *
+_PyObjectStackChunk_New(void);
+
+extern void
+_PyObjectStackChunk_Free(_PyObjectStackChunk *);
+
+extern void
+_PyObjectStackChunk_ClearFreeList(_PyFreeListState *state, int is_finalization);
+
+// Push an item onto the stack. Return -1 on allocation failure, 0 on success.
+static inline int
+_PyObjectStack_Push(_PyObjectStack *stack, PyObject *obj)
+{
+    _PyObjectStackChunk *buf = stack->head;
+    if (buf == NULL || buf->n == _Py_OBJECT_STACK_CHUNK_SIZE) {
+        buf = _PyObjectStackChunk_New();
+        if (buf == NULL) {
+            return -1;
+        }
+        buf->prev = stack->head;
+        buf->n = 0;
+        stack->head = buf;
+    }
+
+    assert(buf->n >= 0 && buf->n < _Py_OBJECT_STACK_CHUNK_SIZE);
+    buf->objs[buf->n] = obj;
+    buf->n++;
+    return 0;
+}
+
+// Pop the top item from the stack.  Return NULL if the stack is empty.
+static inline PyObject *
+_PyObjectStack_Pop(_PyObjectStack *stack)
+{
+    _PyObjectStackChunk *buf = stack->head;
+    if (buf == NULL) {
+        return NULL;
+    }
+    assert(buf->n > 0 && buf->n <= _Py_OBJECT_STACK_CHUNK_SIZE);
+    buf->n--;
+    PyObject *obj = buf->objs[buf->n];
+    if (buf->n == 0) {
+        stack->head = buf->prev;
+        _PyObjectStackChunk_Free(buf);
+    }
+    return obj;
+}
+
+// Remove all items from the stack
+extern void
+_PyObjectStack_Clear(_PyObjectStack *stack);
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_OBJECT_STACK_H
diff --git a/Include/object.h b/Include/object.h
index 48f1ddf..ef3fb72 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -212,7 +212,9 @@ struct _object {
 struct _PyMutex { uint8_t v; };
 
 struct _object {
-    uintptr_t ob_tid;           // thread id (or zero)
+    // ob_tid stores the thread id (or zero). It is also used by the GC to
+    // store linked lists and the computed "gc_refs" refcount.
+    uintptr_t ob_tid;
     uint16_t _padding;
     struct _PyMutex ob_mutex;   // per-object lock
     uint8_t ob_gc_bits;         // gc-related state