gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424)

For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12.

A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.

1 files changed, 55 insertions, 665 deletions

diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index c6ed7da..143b261 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -7,7 +7,6 @@
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_namespace.h"     //_PyNamespace_New()
 #include "pycore_pyerrors.h"      // _PyErr_Clear()
-#include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_typeobject.h"    // _PyType_GetModuleName()
 #include "pycore_weakref.h"       // _PyWeakref_GET_REF()
 
@@ -16,47 +15,12 @@
 /* exceptions */
 /**************/
 
-/* InterpreterError extends Exception */
-
-static PyTypeObject _PyExc_InterpreterError = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "InterpreterError",
-    .tp_doc = PyDoc_STR("An interpreter was not found."),
-    //.tp_base = (PyTypeObject *)PyExc_BaseException,
-};
-PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError;
-
-/* InterpreterNotFoundError extends InterpreterError */
-
-static PyTypeObject _PyExc_InterpreterNotFoundError = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "InterpreterNotFoundError",
-    .tp_doc = PyDoc_STR("An interpreter was not found."),
-    .tp_base = &_PyExc_InterpreterError,
-};
-PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError;
-
-/* lifecycle */
-
-static int
-init_exceptions(PyInterpreterState *interp)
-{
-    _PyExc_InterpreterError.tp_base = (PyTypeObject *)PyExc_BaseException;
-    if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterError) < 0) {
-        return -1;
-    }
-    if (_PyStaticType_InitBuiltin(interp, &_PyExc_InterpreterNotFoundError) < 0) {
-        return -1;
-    }
-    return 0;
-}
-
-static void
-fini_exceptions(PyInterpreterState *interp)
-{
-    _PyStaticType_Dealloc(interp, &_PyExc_InterpreterNotFoundError);
-    _PyStaticType_Dealloc(interp, &_PyExc_InterpreterError);
-}
+static int init_exceptions(PyInterpreterState *);
+static void fini_exceptions(PyInterpreterState *);
+static int _init_not_shareable_error_type(PyInterpreterState *);
+static void _fini_not_shareable_error_type(PyInterpreterState *);
+static PyObject * _get_not_shareable_error_type(PyInterpreterState *);
+#include "crossinterp_exceptions.h"
 
 
 /***************************/
@@ -67,7 +31,7 @@ int
 _Py_CallInInterpreter(PyInterpreterState *interp,
                       _Py_simple_func func, void *arg)
 {
-    if (interp == _PyThreadState_GetCurrent()->interp) {
+    if (interp == PyInterpreterState_Get()) {
         return func(arg);
     }
     // XXX Emit a warning if this fails?
@@ -79,7 +43,7 @@ int
 _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp,
                                 _Py_simple_func func, void *arg)
 {
-    if (interp == _PyThreadState_GetCurrent()->interp) {
+    if (interp == PyInterpreterState_Get()) {
         int res = func(arg);
         PyMem_RawFree(arg);
         return res;
@@ -94,6 +58,20 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp,
 /* cross-interpreter data */
 /**************************/
 
+/* registry of {type -> crossinterpdatafunc} */
+
+/* For now we use a global registry of shareable classes.  An
+   alternative would be to add a tp_* slot for a class's
+   crossinterpdatafunc. It would be simpler and more efficient. */
+
+static void xid_lookup_init(PyInterpreterState *);
+static void xid_lookup_fini(PyInterpreterState *);
+static crossinterpdatafunc lookup_getdata(PyInterpreterState *, PyObject *);
+#include "crossinterp_data_lookup.h"
+
+
+/* lifecycle */
+
 _PyCrossInterpreterData *
 _PyCrossInterpreterData_New(void)
 {
@@ -114,38 +92,6 @@ _PyCrossInterpreterData_Free(_PyCrossInterpreterData *xid)
 }
 
 
-/* exceptions */
-
-static PyStatus
-_init_not_shareable_error_type(PyInterpreterState *interp)
-{
-    const char *name = "_interpreters.NotShareableError";
-    PyObject *base = PyExc_ValueError;
-    PyObject *ns = NULL;
-    PyObject *exctype = PyErr_NewException(name, base, ns);
-    if (exctype == NULL) {
-        PyErr_Clear();
-        return _PyStatus_ERR("could not initialize NotShareableError");
-    }
-
-    interp->xi.PyExc_NotShareableError = exctype;
-    return _PyStatus_OK();
-}
-
-static void
-_fini_not_shareable_error_type(PyInterpreterState *interp)
-{
-    Py_CLEAR(interp->xi.PyExc_NotShareableError);
-}
-
-static PyObject *
-_get_not_shareable_error_type(PyInterpreterState *interp)
-{
-    assert(interp->xi.PyExc_NotShareableError != NULL);
-    return interp->xi.PyExc_NotShareableError;
-}
-
-
 /* defining cross-interpreter data */
 
 static inline void
@@ -156,7 +102,7 @@ _xidata_init(_PyCrossInterpreterData *data)
     assert(data->data == NULL);
     assert(data->obj == NULL);
     *data = (_PyCrossInterpreterData){0};
-    data->interpid = -1;
+    _PyCrossInterpreterData_INTERPID(data) = -1;
 }
 
 static inline void
@@ -193,7 +139,9 @@ _PyCrossInterpreterData_Init(_PyCrossInterpreterData *data,
     // Ideally every object would know its owning interpreter.
     // Until then, we have to rely on the caller to identify it
     // (but we don't need it in all cases).
-    data->interpid = (interp != NULL) ? interp->id : -1;
+    _PyCrossInterpreterData_INTERPID(data) = (interp != NULL)
+        ? PyInterpreterState_GetID(interp)
+        : -1;
     data->new_object = new_object;
 }
 
@@ -223,8 +171,8 @@ _PyCrossInterpreterData_Clear(PyInterpreterState *interp,
     assert(data != NULL);
     // This must be called in the owning interpreter.
     assert(interp == NULL
-           || data->interpid == -1
-           || data->interpid == interp->id);
+           || _PyCrossInterpreterData_INTERPID(data) == -1
+           || _PyCrossInterpreterData_INTERPID(data) == PyInterpreterState_GetID(interp));
     _xidata_clear(data);
 }
 
@@ -238,13 +186,13 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data)
 
     // data->obj may be NULL, so we don't check it.
 
-    if (data->interpid < 0) {
-        _PyErr_SetString(tstate, PyExc_SystemError, "missing interp");
+    if (_PyCrossInterpreterData_INTERPID(data) < 0) {
+        PyErr_SetString(PyExc_SystemError, "missing interp");
         return -1;
     }
 
     if (data->new_object == NULL) {
-        _PyErr_SetString(tstate, PyExc_SystemError, "missing new_object func");
+        PyErr_SetString(PyExc_SystemError, "missing new_object func");
         return -1;
     }
 
@@ -253,25 +201,6 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data)
     return 0;
 }
 
-static crossinterpdatafunc _lookup_getdata_from_registry(
-                                            PyInterpreterState *, PyObject *);
-
-static crossinterpdatafunc
-_lookup_getdata(PyInterpreterState *interp, PyObject *obj)
-{
-   /* Cross-interpreter objects are looked up by exact match on the class.
-      We can reassess this policy when we move from a global registry to a
-      tp_* slot. */
-    return _lookup_getdata_from_registry(interp, obj);
-}
-
-crossinterpdatafunc
-_PyCrossInterpreterData_Lookup(PyObject *obj)
-{
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    return _lookup_getdata(interp, obj);
-}
-
 static inline void
 _set_xid_lookup_failure(PyInterpreterState *interp,
                         PyObject *obj, const char *msg)
@@ -295,8 +224,8 @@ _set_xid_lookup_failure(PyInterpreterState *interp,
 int
 _PyObject_CheckCrossInterpreterData(PyObject *obj)
 {
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    crossinterpdatafunc getdata = _lookup_getdata(interp, obj);
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    crossinterpdatafunc getdata = lookup_getdata(interp, obj);
     if (getdata == NULL) {
         if (!PyErr_Occurred()) {
             _set_xid_lookup_failure(interp, obj, NULL);
@@ -309,20 +238,16 @@ _PyObject_CheckCrossInterpreterData(PyObject *obj)
 int
 _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
 {
-    PyThreadState *tstate = _PyThreadState_GetCurrent();
-#ifdef Py_DEBUG
-    // The caller must hold the GIL
-    _Py_EnsureTstateNotNULL(tstate);
-#endif
+    PyThreadState *tstate = PyThreadState_Get();
     PyInterpreterState *interp = tstate->interp;
 
     // Reset data before re-populating.
     *data = (_PyCrossInterpreterData){0};
-    data->interpid = -1;
+    _PyCrossInterpreterData_INTERPID(data) = -1;
 
     // Call the "getdata" func for the object.
     Py_INCREF(obj);
-    crossinterpdatafunc getdata = _lookup_getdata(interp, obj);
+    crossinterpdatafunc getdata = lookup_getdata(interp, obj);
     if (getdata == NULL) {
         Py_DECREF(obj);
         if (!PyErr_Occurred()) {
@@ -337,7 +262,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data)
     }
 
     // Fill in the blanks and validate the result.
-    data->interpid = interp->id;
+    _PyCrossInterpreterData_INTERPID(data) = PyInterpreterState_GetID(interp);
     if (_check_xidata(tstate, data) != 0) {
         (void)_PyCrossInterpreterData_Release(data);
         return -1;
@@ -374,7 +299,8 @@ _xidata_release(_PyCrossInterpreterData *data, int rawfree)
     }
 
     // Switch to the original interpreter.
-    PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interpid);
+    PyInterpreterState *interp = _PyInterpreterState_LookUpID(
+                                    _PyCrossInterpreterData_INTERPID(data));
     if (interp == NULL) {
         // The interpreter was already destroyed.
         // This function shouldn't have been called.
@@ -408,538 +334,6 @@ _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data)
 }
 
 
-/* registry of {type -> crossinterpdatafunc} */
-
-/* For now we use a global registry of shareable classes.  An
-   alternative would be to add a tp_* slot for a class's
-   crossinterpdatafunc. It would be simpler and more efficient. */
-
-static inline struct _xidregistry *
-_get_global_xidregistry(_PyRuntimeState *runtime)
-{
-    return &runtime->xi.registry;
-}
-
-static inline struct _xidregistry *
-_get_xidregistry(PyInterpreterState *interp)
-{
-    return &interp->xi.registry;
-}
-
-static inline struct _xidregistry *
-_get_xidregistry_for_type(PyInterpreterState *interp, PyTypeObject *cls)
-{
-    struct _xidregistry *registry = _get_global_xidregistry(interp->runtime);
-    if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) {
-        registry = _get_xidregistry(interp);
-    }
-    return registry;
-}
-
-static int
-_xidregistry_add_type(struct _xidregistry *xidregistry,
-                      PyTypeObject *cls, crossinterpdatafunc getdata)
-{
-    struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
-    if (newhead == NULL) {
-        return -1;
-    }
-    *newhead = (struct _xidregitem){
-        // We do not keep a reference, to avoid keeping the class alive.
-        .cls = cls,
-        .refcount = 1,
-        .getdata = getdata,
-    };
-    if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) {
-        // XXX Assign a callback to clear the entry from the registry?
-        newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL);
-        if (newhead->weakref == NULL) {
-            PyMem_RawFree(newhead);
-            return -1;
-        }
-    }
-    newhead->next = xidregistry->head;
-    if (newhead->next != NULL) {
-        newhead->next->prev = newhead;
-    }
-    xidregistry->head = newhead;
-    return 0;
-}
-
-static struct _xidregitem *
-_xidregistry_remove_entry(struct _xidregistry *xidregistry,
-                          struct _xidregitem *entry)
-{
-    struct _xidregitem *next = entry->next;
-    if (entry->prev != NULL) {
-        assert(entry->prev->next == entry);
-        entry->prev->next = next;
-    }
-    else {
-        assert(xidregistry->head == entry);
-        xidregistry->head = next;
-    }
-    if (next != NULL) {
-        next->prev = entry->prev;
-    }
-    Py_XDECREF(entry->weakref);
-    PyMem_RawFree(entry);
-    return next;
-}
-
-static void
-_xidregistry_clear(struct _xidregistry *xidregistry)
-{
-    struct _xidregitem *cur = xidregistry->head;
-    xidregistry->head = NULL;
-    while (cur != NULL) {
-        struct _xidregitem *next = cur->next;
-        Py_XDECREF(cur->weakref);
-        PyMem_RawFree(cur);
-        cur = next;
-    }
-}
-
-static void
-_xidregistry_lock(struct _xidregistry *registry)
-{
-    if (registry->global) {
-        PyMutex_Lock(&registry->mutex);
-    }
-    // else: Within an interpreter we rely on the GIL instead of a separate lock.
-}
-
-static void
-_xidregistry_unlock(struct _xidregistry *registry)
-{
-    if (registry->global) {
-        PyMutex_Unlock(&registry->mutex);
-    }
-}
-
-static struct _xidregitem *
-_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls)
-{
-    struct _xidregitem *cur = xidregistry->head;
-    while (cur != NULL) {
-        if (cur->weakref != NULL) {
-            // cur is/was a heap type.
-            PyObject *registered = _PyWeakref_GET_REF(cur->weakref);
-            if (registered == NULL) {
-                // The weakly ref'ed object was freed.
-                cur = _xidregistry_remove_entry(xidregistry, cur);
-                continue;
-            }
-            assert(PyType_Check(registered));
-            assert(cur->cls == (PyTypeObject *)registered);
-            assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
-            Py_DECREF(registered);
-        }
-        if (cur->cls == cls) {
-            return cur;
-        }
-        cur = cur->next;
-    }
-    return NULL;
-}
-
-int
-_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls,
-                                      crossinterpdatafunc getdata)
-{
-    if (!PyType_Check(cls)) {
-        PyErr_Format(PyExc_ValueError, "only classes may be registered");
-        return -1;
-    }
-    if (getdata == NULL) {
-        PyErr_Format(PyExc_ValueError, "missing 'getdata' func");
-        return -1;
-    }
-
-    int res = 0;
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls);
-    _xidregistry_lock(xidregistry);
-
-    struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls);
-    if (matched != NULL) {
-        assert(matched->getdata == getdata);
-        matched->refcount += 1;
-        goto finally;
-    }
-
-    res = _xidregistry_add_type(xidregistry, cls, getdata);
-
-finally:
-    _xidregistry_unlock(xidregistry);
-    return res;
-}
-
-int
-_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls)
-{
-    int res = 0;
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls);
-    _xidregistry_lock(xidregistry);
-
-    struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls);
-    if (matched != NULL) {
-        assert(matched->refcount > 0);
-        matched->refcount -= 1;
-        if (matched->refcount == 0) {
-            (void)_xidregistry_remove_entry(xidregistry, matched);
-        }
-        res = 1;
-    }
-
-    _xidregistry_unlock(xidregistry);
-    return res;
-}
-
-static crossinterpdatafunc
-_lookup_getdata_from_registry(PyInterpreterState *interp, PyObject *obj)
-{
-    PyTypeObject *cls = Py_TYPE(obj);
-
-    struct _xidregistry *xidregistry = _get_xidregistry_for_type(interp, cls);
-    _xidregistry_lock(xidregistry);
-
-    struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls);
-    crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL;
-
-    _xidregistry_unlock(xidregistry);
-    return func;
-}
-
-/* cross-interpreter data for builtin types */
-
-// bytes
-
-struct _shared_bytes_data {
-    char *bytes;
-    Py_ssize_t len;
-};
-
-static PyObject *
-_new_bytes_object(_PyCrossInterpreterData *data)
-{
-    struct _shared_bytes_data *shared = (struct _shared_bytes_data *)(data->data);
-    return PyBytes_FromStringAndSize(shared->bytes, shared->len);
-}
-
-static int
-_bytes_shared(PyThreadState *tstate, PyObject *obj,
-              _PyCrossInterpreterData *data)
-{
-    if (_PyCrossInterpreterData_InitWithSize(
-            data, tstate->interp, sizeof(struct _shared_bytes_data), obj,
-            _new_bytes_object
-            ) < 0)
-    {
-        return -1;
-    }
-    struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data;
-    if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) {
-        _PyCrossInterpreterData_Clear(tstate->interp, data);
-        return -1;
-    }
-    return 0;
-}
-
-// str
-
-struct _shared_str_data {
-    int kind;
-    const void *buffer;
-    Py_ssize_t len;
-};
-
-static PyObject *
-_new_str_object(_PyCrossInterpreterData *data)
-{
-    struct _shared_str_data *shared = (struct _shared_str_data *)(data->data);
-    return PyUnicode_FromKindAndData(shared->kind, shared->buffer, shared->len);
-}
-
-static int
-_str_shared(PyThreadState *tstate, PyObject *obj,
-            _PyCrossInterpreterData *data)
-{
-    if (_PyCrossInterpreterData_InitWithSize(
-            data, tstate->interp, sizeof(struct _shared_str_data), obj,
-            _new_str_object
-            ) < 0)
-    {
-        return -1;
-    }
-    struct _shared_str_data *shared = (struct _shared_str_data *)data->data;
-    shared->kind = PyUnicode_KIND(obj);
-    shared->buffer = PyUnicode_DATA(obj);
-    shared->len = PyUnicode_GET_LENGTH(obj);
-    return 0;
-}
-
-// int
-
-static PyObject *
-_new_long_object(_PyCrossInterpreterData *data)
-{
-    return PyLong_FromSsize_t((Py_ssize_t)(data->data));
-}
-
-static int
-_long_shared(PyThreadState *tstate, PyObject *obj,
-             _PyCrossInterpreterData *data)
-{
-    /* Note that this means the size of shareable ints is bounded by
-     * sys.maxsize.  Hence on 32-bit architectures that is half the
-     * size of maximum shareable ints on 64-bit.
-     */
-    Py_ssize_t value = PyLong_AsSsize_t(obj);
-    if (value == -1 && PyErr_Occurred()) {
-        if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-            PyErr_SetString(PyExc_OverflowError, "try sending as bytes");
-        }
-        return -1;
-    }
-    _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL,
-            _new_long_object);
-    // data->obj and data->free remain NULL
-    return 0;
-}
-
-// float
-
-static PyObject *
-_new_float_object(_PyCrossInterpreterData *data)
-{
-    double * value_ptr = data->data;
-    return PyFloat_FromDouble(*value_ptr);
-}
-
-static int
-_float_shared(PyThreadState *tstate, PyObject *obj,
-             _PyCrossInterpreterData *data)
-{
-    if (_PyCrossInterpreterData_InitWithSize(
-            data, tstate->interp, sizeof(double), NULL,
-            _new_float_object
-            ) < 0)
-    {
-        return -1;
-    }
-    double *shared = (double *)data->data;
-    *shared = PyFloat_AsDouble(obj);
-    return 0;
-}
-
-// None
-
-static PyObject *
-_new_none_object(_PyCrossInterpreterData *data)
-{
-    // XXX Singleton refcounts are problematic across interpreters...
-    return Py_NewRef(Py_None);
-}
-
-static int
-_none_shared(PyThreadState *tstate, PyObject *obj,
-             _PyCrossInterpreterData *data)
-{
-    _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL,
-            _new_none_object);
-    // data->data, data->obj and data->free remain NULL
-    return 0;
-}
-
-// bool
-
-static PyObject *
-_new_bool_object(_PyCrossInterpreterData *data)
-{
-    if (data->data){
-        Py_RETURN_TRUE;
-    }
-    Py_RETURN_FALSE;
-}
-
-static int
-_bool_shared(PyThreadState *tstate, PyObject *obj,
-             _PyCrossInterpreterData *data)
-{
-    _PyCrossInterpreterData_Init(data, tstate->interp,
-            (void *) (Py_IsTrue(obj) ? (uintptr_t) 1 : (uintptr_t) 0), NULL,
-            _new_bool_object);
-    // data->obj and data->free remain NULL
-    return 0;
-}
-
-// tuple
-
-struct _shared_tuple_data {
-    Py_ssize_t len;
-    _PyCrossInterpreterData **data;
-};
-
-static PyObject *
-_new_tuple_object(_PyCrossInterpreterData *data)
-{
-    struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data->data);
-    PyObject *tuple = PyTuple_New(shared->len);
-    if (tuple == NULL) {
-        return NULL;
-    }
-
-    for (Py_ssize_t i = 0; i < shared->len; i++) {
-        PyObject *item = _PyCrossInterpreterData_NewObject(shared->data[i]);
-        if (item == NULL){
-            Py_DECREF(tuple);
-            return NULL;
-        }
-        PyTuple_SET_ITEM(tuple, i, item);
-    }
-    return tuple;
-}
-
-static void
-_tuple_shared_free(void* data)
-{
-    struct _shared_tuple_data *shared = (struct _shared_tuple_data *)(data);
-#ifndef NDEBUG
-    int64_t interpid = PyInterpreterState_GetID(_PyInterpreterState_GET());
-#endif
-    for (Py_ssize_t i = 0; i < shared->len; i++) {
-        if (shared->data[i] != NULL) {
-            assert(shared->data[i]->interpid == interpid);
-            _PyCrossInterpreterData_Release(shared->data[i]);
-            PyMem_RawFree(shared->data[i]);
-            shared->data[i] = NULL;
-        }
-    }
-    PyMem_Free(shared->data);
-    PyMem_RawFree(shared);
-}
-
-static int
-_tuple_shared(PyThreadState *tstate, PyObject *obj,
-             _PyCrossInterpreterData *data)
-{
-    Py_ssize_t len = PyTuple_GET_SIZE(obj);
-    if (len < 0) {
-        return -1;
-    }
-    struct _shared_tuple_data *shared = PyMem_RawMalloc(sizeof(struct _shared_tuple_data));
-    if (shared == NULL){
-        PyErr_NoMemory();
-        return -1;
-    }
-
-    shared->len = len;
-    shared->data = (_PyCrossInterpreterData **) PyMem_Calloc(shared->len, sizeof(_PyCrossInterpreterData *));
-    if (shared->data == NULL) {
-        PyErr_NoMemory();
-        return -1;
-    }
-
-    for (Py_ssize_t i = 0; i < shared->len; i++) {
-        _PyCrossInterpreterData *data = _PyCrossInterpreterData_New();
-        if (data == NULL) {
-            goto error;  // PyErr_NoMemory already set
-        }
-        PyObject *item = PyTuple_GET_ITEM(obj, i);
-
-        int res = -1;
-        if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) {
-            res = _PyObject_GetCrossInterpreterData(item, data);
-            _Py_LeaveRecursiveCallTstate(tstate);
-        }
-        if (res < 0) {
-            PyMem_RawFree(data);
-            goto error;
-        }
-        shared->data[i] = data;
-    }
-    _PyCrossInterpreterData_Init(
-            data, tstate->interp, shared, obj, _new_tuple_object);
-    data->free = _tuple_shared_free;
-    return 0;
-
-error:
-    _tuple_shared_free(shared);
-    return -1;
-}
-
-// registration
-
-static void
-_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
-{
-    // None
-    if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
-        Py_FatalError("could not register None for cross-interpreter sharing");
-    }
-
-    // int
-    if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) {
-        Py_FatalError("could not register int for cross-interpreter sharing");
-    }
-
-    // bytes
-    if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) {
-        Py_FatalError("could not register bytes for cross-interpreter sharing");
-    }
-
-    // str
-    if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) {
-        Py_FatalError("could not register str for cross-interpreter sharing");
-    }
-
-    // bool
-    if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) {
-        Py_FatalError("could not register bool for cross-interpreter sharing");
-    }
-
-    // float
-    if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) {
-        Py_FatalError("could not register float for cross-interpreter sharing");
-    }
-
-    // tuple
-    if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) {
-        Py_FatalError("could not register tuple for cross-interpreter sharing");
-    }
-}
-
-/* registry lifecycle */
-
-static void
-_xidregistry_init(struct _xidregistry *registry)
-{
-    if (registry->initialized) {
-        return;
-    }
-    registry->initialized = 1;
-
-    if (registry->global) {
-        // Registering the builtins is cheap so we don't bother doing it lazily.
-        assert(registry->head == NULL);
-        _register_builtins_for_crossinterpreter_data(registry);
-    }
-}
-
-static void
-_xidregistry_fini(struct _xidregistry *registry)
-{
-    if (!registry->initialized) {
-        return;
-    }
-    registry->initialized = 0;
-
-    _xidregistry_clear(registry);
-}
-
-
 /*************************/
 /* convenience utilities */
 /*************************/
@@ -1023,6 +417,10 @@ error:
     return -1;
 }
 
+// We accommodate backports here.
+#ifndef _Py_EMPTY_STR
+# define _Py_EMPTY_STR &_Py_STR(empty)
+#endif
 
 static const char *
 _format_TracebackException(PyObject *tbexc)
@@ -1031,7 +429,8 @@ _format_TracebackException(PyObject *tbexc)
     if (lines == NULL) {
         return NULL;
     }
-    PyObject *formatted_obj = PyUnicode_Join(&_Py_STR(empty), lines);
+    assert(_Py_EMPTY_STR != NULL);
+    PyObject *formatted_obj = PyUnicode_Join(_Py_EMPTY_STR, lines);
     Py_DECREF(lines);
     if (formatted_obj == NULL) {
         return NULL;
@@ -1600,7 +999,7 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid)
         return 0;
     }
     if (p_interpid != NULL) {
-        *p_interpid = item->data->interpid;
+        *p_interpid = _PyCrossInterpreterData_INTERPID(item->data);
     }
     return 1;
 }
@@ -1874,7 +1273,7 @@ _PyXI_FreeNamespace(_PyXI_namespace *ns)
         return;
     }
 
-    if (interpid == PyInterpreterState_GetID(_PyInterpreterState_GET())) {
+    if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) {
         _sharedns_free(ns);
     }
     else {
@@ -2014,7 +1413,7 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp)
     PyThreadState *prev = tstate;
     if (interp != tstate->interp) {
         tstate = PyThreadState_New(interp);
-        tstate->_whence = _PyThreadState_WHENCE_EXEC;
+        _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC);
         // XXX Possible GILState issues?
         session->prev_tstate = PyThreadState_Swap(tstate);
         assert(session->prev_tstate == prev);
@@ -2073,7 +1472,7 @@ _propagate_not_shareable_error(_PyXI_session *session)
     if (session == NULL) {
         return;
     }
-    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyInterpreterState *interp = PyInterpreterState_Get();
     if (PyErr_ExceptionMatches(_get_not_shareable_error_type(interp))) {
         // We want to propagate the exception directly.
         session->_error_override = _PyXI_ERR_NOT_SHAREABLE;
@@ -2245,18 +1644,12 @@ _PyXI_Exit(_PyXI_session *session)
 PyStatus
 _PyXI_Init(PyInterpreterState *interp)
 {
-    PyStatus status;
-
-    // Initialize the XID registry.
-    if (_Py_IsMainInterpreter(interp)) {
-        _xidregistry_init(_get_global_xidregistry(interp->runtime));
-    }
-    _xidregistry_init(_get_xidregistry(interp));
+    // Initialize the XID lookup state (e.g. registry).
+    xid_lookup_init(interp);
 
     // Initialize exceptions (heap types).
-    status = _init_not_shareable_error_type(interp);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (_init_not_shareable_error_type(interp) < 0) {
+        return _PyStatus_ERR("failed to initialize NotShareableError");
     }
 
     return _PyStatus_OK();
@@ -2271,11 +1664,8 @@ _PyXI_Fini(PyInterpreterState *interp)
     // Finalize exceptions (heap types).
     _fini_not_shareable_error_type(interp);
 
-    // Finalize the XID registry.
-    _xidregistry_fini(_get_xidregistry(interp));
-    if (_Py_IsMainInterpreter(interp)) {
-        _xidregistry_fini(_get_global_xidregistry(interp->runtime));
-    }
+    // Finalize the XID lookup state (e.g. registry).
+    xid_lookup_fini(interp);
 }
 
 PyStatus