gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424)

For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12.

A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.

4 files changed, 74 insertions, 50 deletions

diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h
new file mode 100644
index 0000000..5661a26
--- /dev/null
+++ b/Modules/_interpreters_common.h
@@ -0,0 +1,13 @@
+
+#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
+    PyInit_ ## NAME
+#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
+    _RESOLVE_MODINIT_FUNC_NAME(NAME)
+
+
+static int
+ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
+{
+    //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
+    return _PyCrossInterpreterData_RegisterClass(cls, getdata);
+}
diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c
index 4e9b8a8..a2974ac 100644
--- a/Modules/_xxinterpchannelsmodule.c
+++ b/Modules/_xxinterpchannelsmodule.c
@@ -17,6 +17,8 @@
 #include <sched.h>          // sched_yield()
 #endif
 
+#include "_interpreters_common.h"
+
 
 /*
 This module has the following process-global state:
@@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
 API..  The module does not create any objects that are shared globally.
 */
 
-#define MODULE_NAME "_xxinterpchannels"
+#define MODULE_NAME _xxinterpchannels
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -101,7 +105,7 @@ static int
 register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
                    struct xid_class_registry *classes)
 {
-    int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
+    int res = ensure_xid_class(cls, shared);
     if (res == 0) {
         assert(classes->count < MAX_XID_CLASSES);
         // The class has refs elsewhere, so we need to incref here.
@@ -167,7 +171,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
-    PyObject *name = PyUnicode_FromString(MODULE_NAME);
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
     if (name == NULL) {
         return NULL;
     }
@@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
 }
 
 #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
-    add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
+    add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
 
 static PyTypeObject *
 add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@@ -299,7 +303,7 @@ _get_current_module_state(void)
     if (mod == NULL) {
         // XXX import it?
         PyErr_SetString(PyExc_RuntimeError,
-                        MODULE_NAME " module not imported yet");
+                        MODULE_NAME_STR " module not imported yet");
         return NULL;
     }
     module_state *state = get_module_state(mod);
@@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
     while (next != NULL) {
         _channelitem *item = next;
         next = item->next;
-        if (item->data->interpid == interpid) {
+        if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
             if (prev == NULL) {
                 queue->first = item->next;
             }
@@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = {
 };
 
 static PyStructSequence_Desc channel_info_desc = {
-    .name = MODULE_NAME ".ChannelInfo",
+    .name = MODULE_NAME_STR ".ChannelInfo",
     .doc = channel_info_doc,
     .fields = channel_info_fields,
     .n_in_sequence = 8,
@@ -2474,10 +2478,11 @@ struct _channelid_xid {
 static PyObject *
 _channelid_from_xid(_PyCrossInterpreterData *data)
 {
-    struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+    struct _channelid_xid *xid = \
+                (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
 
     // It might not be imported yet, so we can't use _get_current_module().
-    PyObject *mod = PyImport_ImportModule(MODULE_NAME);
+    PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
     if (mod == NULL) {
         return NULL;
     }
@@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
     {
         return -1;
     }
-    struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+    struct _channelid_xid *xid = \
+                (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
     xid->cid = ((channelid *)obj)->cid;
     xid->end = ((channelid *)obj)->end;
     xid->resolve = ((channelid *)obj)->resolve;
@@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = {
 };
 
 static PyType_Spec channelid_typespec = {
-    .name = MODULE_NAME ".ChannelID",
+    .name = MODULE_NAME_STR ".ChannelID",
     .basicsize = sizeof(channelid),
     .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
               Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
     if (res < 0) {
         return -1;
     }
-    data->new_object = _channelend_from_xid;
+    _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
     return 0;
 }
 
@@ -3379,7 +3385,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = MODULE_NAME,
+    .m_name = MODULE_NAME_STR,
     .m_doc = module_doc,
     .m_size = sizeof(module_state),
     .m_methods = module_functions,
@@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = {
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpchannels(void)
+MODINIT_FUNC_NAME(void)
 {
     return PyModuleDef_Init(&moduledef);
 }
diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c
index 537ba91..7d8c67f 100644
--- a/Modules/_xxinterpqueuesmodule.c
+++ b/Modules/_xxinterpqueuesmodule.c
@@ -8,8 +8,12 @@
 #include "Python.h"
 #include "pycore_crossinterp.h"   // struct _xid
 
+#include "_interpreters_common.h"
 
-#define MODULE_NAME "_xxinterpqueues"
+
+#define MODULE_NAME _xxinterpqueues
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -64,7 +68,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
-    PyObject *name = PyUnicode_FromString(MODULE_NAME);
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
     if (name == NULL) {
         return NULL;
     }
@@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
     while (next != NULL) {
         _queueitem *item = next;
         next = item->next;
-        if (item->data->interpid == interpid) {
+        if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
             if (prev == NULL) {
                 queue->items.first = item->next;
             }
@@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
     }
     state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
 
-    if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) {
+    if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
         return -1;
     }
 
@@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
 static PyObject *
 _queueobj_from_xid(_PyCrossInterpreterData *data)
 {
-    int64_t qid = *(int64_t *)data->data;
+    int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
     PyObject *qidobj = PyLong_FromLongLong(qid);
     if (qidobj == NULL) {
         return NULL;
@@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
     if (mod == NULL) {
         // XXX import it?
         PyErr_SetString(PyExc_RuntimeError,
-                        MODULE_NAME " module not imported yet");
+                        MODULE_NAME_STR " module not imported yet");
         return NULL;
     }
 
@@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
     _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
                                  _queueobj_from_xid);
     Py_DECREF(qidobj);
-    data->free = _queueid_xid_free;
+    _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
     return 0;
 }
 
@@ -1670,7 +1674,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = MODULE_NAME,
+    .m_name = MODULE_NAME_STR,
     .m_doc = module_doc,
     .m_size = sizeof(module_state),
     .m_methods = module_functions,
@@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef = {
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpqueues(void)
+MODINIT_FUNC_NAME(void)
 {
     return PyModuleDef_Init(&moduledef);
 }
diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index 4e9e134..b4004d1 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -19,8 +19,12 @@
 #include "interpreteridobject.h"
 #include "marshal.h"              // PyMarshal_ReadObjectFromString()
 
+#include "_interpreters_common.h"
 
-#define MODULE_NAME "_xxsubinterpreters"
+
+#define MODULE_NAME _xxsubinterpreters
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 static PyInterpreterState *
@@ -125,7 +129,7 @@ get_interpid_obj(PyInterpreterState *interp)
 static PyObject *
 _get_current_module(void)
 {
-    PyObject *name = PyUnicode_FromString(MODULE_NAME);
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
     if (name == NULL) {
         return NULL;
     }
@@ -152,16 +156,16 @@ typedef struct {
 static PyObject *
 xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data)
 {
-    assert(data->data != NULL);
-    assert(data->obj == NULL);
-    assert(data->interpid >= 0);
+    assert(_PyCrossInterpreterData_DATA(data) != NULL);
+    assert(_PyCrossInterpreterData_OBJ(data) == NULL);
+    assert(_PyCrossInterpreterData_INTERPID(data) >= 0);
     XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject));
     if (self == NULL) {
         return NULL;
     }
     PyObject_Init((PyObject *)self, cls);
-    self->view = (Py_buffer *)data->data;
-    self->interpid = data->interpid;
+    self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data);
+    self->interpid = _PyCrossInterpreterData_INTERPID(data);
     return (PyObject *)self;
 }
 
@@ -209,7 +213,7 @@ static PyType_Slot XIBufferViewType_slots[] = {
 };
 
 static PyType_Spec XIBufferViewType_spec = {
-    .name = MODULE_NAME ".CrossInterpreterBufferView",
+    .name = MODULE_NAME_STR ".CrossInterpreterBufferView",
     .basicsize = sizeof(XIBufferViewObject),
     .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
               Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -267,8 +271,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
     *p_state = cls;
 
     // Register XID for the builtin memoryview type.
-    if (_PyCrossInterpreterData_RegisterClass(
-                &PyMemoryView_Type, _memoryview_shared) < 0) {
+    if (ensure_xid_class(&PyMemoryView_Type, _memoryview_shared) < 0) {
         return -1;
     }
     // We don't ever bother un-registering memoryview.
@@ -303,7 +306,7 @@ _get_current_module_state(void)
     if (mod == NULL) {
         // XXX import it?
         PyErr_SetString(PyExc_RuntimeError,
-                        MODULE_NAME " module not imported yet");
+                        MODULE_NAME_STR " module not imported yet");
         return NULL;
     }
     module_state *state = get_module_state(mod);
@@ -372,9 +375,7 @@ check_code_object(PyCodeObject *code)
     }
     // We trust that no code objects under co_consts have unbound cell vars.
 
-    if (code->co_executors != NULL
-        || code->_co_instrumentation_version > 0)
-    {
+    if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) {
         return "only basic functions are supported";
     }
     if (code->_co_monitoring != NULL) {
@@ -602,7 +603,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
 
     // Destroy the interpreter.
     PyThreadState *tstate = PyThreadState_New(interp);
-    tstate->_whence = _PyThreadState_WHENCE_INTERP;
+    _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
     // XXX Possible GILState issues?
     PyThreadState *save_tstate = PyThreadState_Swap(tstate);
     Py_EndInterpreter(tstate);
@@ -691,7 +692,7 @@ static PyObject *
 interp_set___main___attrs(PyObject *self, PyObject *args)
 {
     PyObject *id, *updates;
-    if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs",
+    if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME_STR ".set___main___attrs",
                           &id, &updates))
     {
         return NULL;
@@ -856,18 +857,18 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *id, *code;
     PyObject *shared = NULL;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO|O:" MODULE_NAME ".exec", kwlist,
+                                     "OO|O:" MODULE_NAME_STR ".exec", kwlist,
                                      &id, &code, &shared)) {
         return NULL;
     }
 
     const char *expected = "a string, a function, or a code object";
     if (PyUnicode_Check(code)) {
-         code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec",
+         code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec",
                                                "argument 2", expected);
     }
     else {
-         code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec",
+         code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec",
                                              "argument 2", expected);
     }
     if (code == NULL) {
@@ -908,12 +909,12 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *id, *script;
     PyObject *shared = NULL;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OU|O:" MODULE_NAME ".run_string", kwlist,
+                                     "OU|O:" MODULE_NAME_STR ".run_string", kwlist,
                                      &id, &script, &shared)) {
         return NULL;
     }
 
-    script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec",
+    script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".exec",
                                             "argument 2", "a string");
     if (script == NULL) {
         return NULL;
@@ -934,7 +935,7 @@ PyDoc_STRVAR(run_string_doc,
 \n\
 Execute the provided string in the identified interpreter.\n\
 \n\
-(See " MODULE_NAME ".exec().");
+(See " MODULE_NAME_STR ".exec().");
 
 static PyObject *
 interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
@@ -943,12 +944,12 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *id, *func;
     PyObject *shared = NULL;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "OO|O:" MODULE_NAME ".run_func", kwlist,
+                                     "OO|O:" MODULE_NAME_STR ".run_func", kwlist,
                                      &id, &func, &shared)) {
         return NULL;
     }
 
-    PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec",
+    PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec",
                                           "argument 2",
                                           "a function or a code object");
     if (code == NULL) {
@@ -972,7 +973,7 @@ Execute the body of the provided function in the identified interpreter.\n\
 Code objects are also supported.  In both cases, closures and args\n\
 are not supported.  Methods and other callables are not supported either.\n\
 \n\
-(See " MODULE_NAME ".exec().");
+(See " MODULE_NAME_STR ".exec().");
 
 
 static PyObject *
@@ -1166,7 +1167,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = MODULE_NAME,
+    .m_name = MODULE_NAME_STR,
     .m_doc = module_doc,
     .m_size = sizeof(module_state),
     .m_methods = module_functions,
@@ -1177,7 +1178,7 @@ static struct PyModuleDef moduledef = {
 };
 
 PyMODINIT_FUNC
-PyInit__xxsubinterpreters(void)
+MODINIT_FUNC_NAME(void)
 {
     return PyModuleDef_Init(&moduledef);
 }