gh-109793: Allow Switching Interpreters During Finalization (gh-109794)

Essentially, we should check the thread ID rather than the thread state pointer.

4 files changed, 55 insertions, 2 deletions

diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
index ab18238..ce23e13 100644
--- a/Include/cpython/pyatomic.h
+++ b/Include/cpython/pyatomic.h
@@ -501,3 +501,20 @@ static inline void _Py_atomic_fence_release(void);
 #else
 #  error "no available pyatomic implementation for this platform/compiler"
 #endif
+
+
+// --- aliases ---------------------------------------------------------------
+
+#if SIZEOF_LONG == 8
+# define _Py_atomic_load_ulong _Py_atomic_load_uint64
+# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint64_relaxed
+# define _Py_atomic_store_ulong _Py_atomic_store_uint64
+# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint64_relaxed
+#elif SIZEOF_LONG == 4
+# define _Py_atomic_load_ulong _Py_atomic_load_uint32
+# define _Py_atomic_load_ulong_relaxed _Py_atomic_load_uint32_relaxed
+# define _Py_atomic_store_ulong _Py_atomic_store_uint32
+# define _Py_atomic_store_ulong_relaxed _Py_atomic_store_uint32_relaxed
+#else
+# error "long must be 4 or 8 bytes in size"
+#endif  // SIZEOF_LONG
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index ba5764e..0912bd1 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -93,6 +93,8 @@ struct _is {
        and _PyInterpreterState_SetFinalizing()
        to access it, don't access it directly. */
     _Py_atomic_address _finalizing;
+    /* The ID of the OS thread in which we are finalizing. */
+    unsigned long _finalizing_id;
 
     struct _gc_runtime_state gc;
 
@@ -215,9 +217,23 @@ _PyInterpreterState_GetFinalizing(PyInterpreterState *interp) {
     return (PyThreadState*)_Py_atomic_load_relaxed(&interp->_finalizing);
 }
 
+static inline unsigned long
+_PyInterpreterState_GetFinalizingID(PyInterpreterState *interp) {
+    return _Py_atomic_load_ulong_relaxed(&interp->_finalizing_id);
+}
+
 static inline void
 _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tstate) {
     _Py_atomic_store_relaxed(&interp->_finalizing, (uintptr_t)tstate);
+    if (tstate == NULL) {
+        _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id, 0);
+    }
+    else {
+        // XXX Re-enable this assert once gh-109860 is fixed.
+        //assert(tstate->thread_id == PyThread_get_thread_ident());
+        _Py_atomic_store_ulong_relaxed(&interp->_finalizing_id,
+                                       tstate->thread_id);
+    }
 }
 
 
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 9fc8ae9..2e568f8 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -36,8 +36,12 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
 static inline int
 _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
 {
-    return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL &&
-            interp == &interp->runtime->_main_interpreter);
+    /* bpo-39877: Access _PyRuntime directly rather than using
+       tstate->interp->runtime to support calls from Python daemon threads.
+       After Py_Finalize() has been called, tstate can be a dangling pointer:
+       point to PyThreadState freed memory. */
+    return (_PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL &&
+            interp == &_PyRuntime._main_interpreter);
 }
 
 
diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h
index 0ddc405..cc3a342 100644
--- a/Include/internal/pycore_runtime.h
+++ b/Include/internal/pycore_runtime.h
@@ -171,6 +171,8 @@ typedef struct pyruntimestate {
        Use _PyRuntimeState_GetFinalizing() and _PyRuntimeState_SetFinalizing()
        to access it, don't access it directly. */
     _Py_atomic_address _finalizing;
+    /* The ID of the OS thread in which we are finalizing. */
+    unsigned long _finalizing_id;
 
     struct pyinterpreters {
         PyThread_type_lock mutex;
@@ -303,9 +305,23 @@ _PyRuntimeState_GetFinalizing(_PyRuntimeState *runtime) {
     return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->_finalizing);
 }
 
+static inline unsigned long
+_PyRuntimeState_GetFinalizingID(_PyRuntimeState *runtime) {
+    return _Py_atomic_load_ulong_relaxed(&runtime->_finalizing_id);
+}
+
 static inline void
 _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) {
     _Py_atomic_store_relaxed(&runtime->_finalizing, (uintptr_t)tstate);
+    if (tstate == NULL) {
+        _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id, 0);
+    }
+    else {
+        // XXX Re-enable this assert once gh-109860 is fixed.
+        //assert(tstate->thread_id == PyThread_get_thread_ident());
+        _Py_atomic_store_ulong_relaxed(&runtime->_finalizing_id,
+                                       tstate->thread_id);
+    }
 }
 
 #ifdef __cplusplus