gh-109549: Add new states to PyThreadState to support PEP 703 (gh-109915)

This adds a new field 'state' to PyThreadState that can take on one of three values: _Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, or _Py_THREAD_GC.  The "attached" and "detached" states correspond closely to acquiring and releasing the GIL.  The "gc" state is current unused, but will be used to implement stop-the-world GC for --disable-gil builds in the near future.

3 files changed, 46 insertions, 1 deletions

diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 8fd0624..40102f8 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -102,6 +102,10 @@ struct _ts {
 #endif
     int _whence;
 
+    /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_GC).
+       See Include/internal/pycore_pystate.h for more details. */
+    int state;
+
     int py_recursion_remaining;
     int py_recursion_limit;
 
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 48fee69..d3ea3a8 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -121,7 +121,6 @@ extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
 extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *);
-extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);
 
 extern void _PyEval_DeactivateOpCache(void);
 
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index a60d949..7135b1e 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -11,6 +11,33 @@ extern "C" {
 #include "pycore_runtime.h"       // _PyRuntime
 
 
+// Values for PyThreadState.state. A thread must be in the "attached" state
+// before calling most Python APIs. If the GIL is enabled, then "attached"
+// implies that the thread holds the GIL and "detached" implies that the
+// thread does not hold the GIL (or is in the process of releasing it). In
+// `--disable-gil` builds, multiple threads may be "attached" to the same
+// interpreter at the same time. Only the "bound" thread may perform the
+// transitions between "attached" and "detached" on its own PyThreadState.
+//
+// The "gc" state is used to implement stop-the-world pauses, such as for
+// cyclic garbage collection. It is only used in `--disable-gil` builds. It is
+// similar to the "detached" state, but only the thread performing a
+// stop-the-world pause may transition threads between the "detached" and "gc"
+// states. A thread trying to "attach" from the "gc" state will block until
+// it is transitioned back to "detached" when the stop-the-world pause is
+// complete.
+//
+// State transition diagram:
+//
+//            (bound thread)        (stop-the-world thread)
+// [attached]       <->       [detached]       <->       [gc]
+//
+// See `_PyThreadState_Attach()` and `_PyThreadState_Detach()`.
+#define _Py_THREAD_DETACHED     0
+#define _Py_THREAD_ATTACHED     1
+#define _Py_THREAD_GC           2
+
+
 /* Check if the current thread is the main thread.
    Use _Py_IsMainInterpreter() to check if it's the main interpreter. */
 static inline int
@@ -104,6 +131,21 @@ _PyThreadState_GET(void)
 #endif
 }
 
+// Attaches the current thread to the interpreter.
+//
+// This may block while acquiring the GIL (if the GIL is enabled) or while
+// waiting for a stop-the-world pause (if the GIL is disabled).
+//
+// High-level code should generally call PyEval_RestoreThread() instead, which
+// calls this function.
+void _PyThreadState_Attach(PyThreadState *tstate);
+
+// Detaches the current thread from the interpreter.
+//
+// High-level code should generally call PyEval_SaveThread() instead, which
+// calls this function.
+void _PyThreadState_Detach(PyThreadState *tstate);
+
 
 static inline void
 _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate)