gh-76785: Add More Tests to test_interpreters.test_api (gh-117662)

In addition to the increase test coverage, this is a precursor to sorting out how we handle interpreters created directly via the C-API.

4 files changed, 299 insertions, 15 deletions

diff --git a/Python/crossinterp.c b/Python/crossinterp.c
index 16efe9c..fb0dae0 100644
--- a/Python/crossinterp.c
+++ b/Python/crossinterp.c
@@ -468,7 +468,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int rawfree)
 /***********************/
 
 static int
-_excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
+_excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc)
 {
     /* Note that this copies directly rather than into an intermediate
        struct and does not clear on error.  If we need that then we
@@ -504,7 +504,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
     }
     info->qualname = _copy_string_obj_raw(strobj, NULL);
     Py_DECREF(strobj);
-    if (info->name == NULL) {
+    if (info->qualname == NULL) {
         return -1;
     }
 
@@ -515,10 +515,51 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
     }
     info->module = _copy_string_obj_raw(strobj, NULL);
     Py_DECREF(strobj);
+    if (info->module == NULL) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int
+_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype)
+{
+    PyObject *strobj = NULL;
+
+    // __name__
+    strobj = PyObject_GetAttrString(exctype, "__name__");
+    if (strobj == NULL) {
+        return -1;
+    }
+    info->name = _copy_string_obj_raw(strobj, NULL);
+    Py_DECREF(strobj);
     if (info->name == NULL) {
         return -1;
     }
 
+    // __qualname__
+    strobj = PyObject_GetAttrString(exctype, "__qualname__");
+    if (strobj == NULL) {
+        return -1;
+    }
+    info->qualname = _copy_string_obj_raw(strobj, NULL);
+    Py_DECREF(strobj);
+    if (info->qualname == NULL) {
+        return -1;
+    }
+
+    // __module__
+    strobj = PyObject_GetAttrString(exctype, "__module__");
+    if (strobj == NULL) {
+        return -1;
+    }
+    info->module = _copy_string_obj_raw(strobj, NULL);
+    Py_DECREF(strobj);
+    if (info->module == NULL) {
+        return -1;
+    }
+
     return 0;
 }
 
@@ -584,7 +625,7 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info)
     *info = (_PyXI_excinfo){{NULL}};
 }
 
-static PyObject *
+PyObject *
 _PyXI_excinfo_format(_PyXI_excinfo *info)
 {
     const char *module, *qualname;
@@ -627,7 +668,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
     }
     const char *failure = NULL;
 
-    if (_excinfo_init_type(&info->type, exc) < 0) {
+    if (_excinfo_init_type_from_exception(&info->type, exc) < 0) {
         failure = "error while initializing exception type snapshot";
         goto error;
     }
@@ -672,6 +713,57 @@ error:
     return failure;
 }
 
+static const char *
+_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj)
+{
+    const char *failure = NULL;
+
+    PyObject *exctype = PyObject_GetAttrString(obj, "type");
+    if (exctype == NULL) {
+        failure = "exception snapshot missing 'type' attribute";
+        goto error;
+    }
+    int res = _excinfo_init_type_from_object(&info->type, exctype);
+    Py_DECREF(exctype);
+    if (res < 0) {
+        failure = "error while initializing exception type snapshot";
+        goto error;
+    }
+
+    // Extract the exception message.
+    PyObject *msgobj = PyObject_GetAttrString(obj, "msg");
+    if (msgobj == NULL) {
+        failure = "exception snapshot missing 'msg' attribute";
+        goto error;
+    }
+    info->msg = _copy_string_obj_raw(msgobj, NULL);
+    Py_DECREF(msgobj);
+    if (info->msg == NULL) {
+        failure = "error while copying exception message";
+        goto error;
+    }
+
+    // Pickle a traceback.TracebackException.
+    PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay");
+    if (errdisplay == NULL) {
+        failure = "exception snapshot missing 'errdisplay' attribute";
+        goto error;
+    }
+    info->errdisplay = _copy_string_obj_raw(errdisplay, NULL);
+    Py_DECREF(errdisplay);
+    if (info->errdisplay == NULL) {
+        failure = "error while copying exception error display";
+        goto error;
+    }
+
+    return NULL;
+
+error:
+    assert(failure != NULL);
+    _PyXI_excinfo_Clear(info);
+    return failure;
+}
+
 static void
 _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype)
 {
@@ -825,6 +917,47 @@ error:
 }
 
 
+int
+_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc)
+{
+    assert(!PyErr_Occurred());
+    if (exc == NULL || exc == Py_None) {
+        PyErr_SetString(PyExc_ValueError, "missing exc");
+        return -1;
+    }
+    const char *failure;
+    if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) {
+        failure = _PyXI_excinfo_InitFromException(info, exc);
+    }
+    else {
+        failure = _PyXI_excinfo_InitFromObject(info, exc);
+    }
+    if (failure != NULL) {
+        PyErr_SetString(PyExc_Exception, failure);
+        return -1;
+    }
+    return 0;
+}
+
+PyObject *
+_PyXI_FormatExcInfo(_PyXI_excinfo *info)
+{
+    return _PyXI_excinfo_format(info);
+}
+
+PyObject *
+_PyXI_ExcInfoAsObject(_PyXI_excinfo *info)
+{
+    return _PyXI_excinfo_AsObject(info);
+}
+
+void
+_PyXI_ClearExcInfo(_PyXI_excinfo *info)
+{
+    _PyXI_excinfo_Clear(info);
+}
+
+
 /***************************/
 /* short-term data sharing */
 /***************************/
@@ -1682,3 +1815,95 @@ _PyXI_FiniTypes(PyInterpreterState *interp)
 {
     fini_exceptions(interp);
 }
+
+
+/*************/
+/* other API */
+/*************/
+
+PyInterpreterState *
+_PyXI_NewInterpreter(PyInterpreterConfig *config,
+                     PyThreadState **p_tstate, PyThreadState **p_save_tstate)
+{
+    PyThreadState *save_tstate = PyThreadState_Swap(NULL);
+    assert(save_tstate != NULL);
+
+    PyThreadState *tstate;
+    PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
+    if (PyStatus_Exception(status)) {
+        // Since no new thread state was created, there is no exception
+        // to propagate; raise a fresh one after swapping back in the
+        // old thread state.
+        PyThreadState_Swap(save_tstate);
+        _PyErr_SetFromPyStatus(status);
+        PyObject *exc = PyErr_GetRaisedException();
+        PyErr_SetString(PyExc_InterpreterError,
+                        "sub-interpreter creation failed");
+        _PyErr_ChainExceptions1(exc);
+        return NULL;
+    }
+    assert(tstate != NULL);
+    PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
+
+    _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_XI);
+
+    if (p_tstate != NULL) {
+        // We leave the new thread state as the current one.
+        *p_tstate = tstate;
+    }
+    else {
+        // Throw away the initial tstate.
+        PyThreadState_Clear(tstate);
+        PyThreadState_Swap(save_tstate);
+        PyThreadState_Delete(tstate);
+        save_tstate = NULL;
+    }
+    if (p_save_tstate != NULL) {
+        *p_save_tstate = save_tstate;
+    }
+    return interp;
+}
+
+void
+_PyXI_EndInterpreter(PyInterpreterState *interp,
+                     PyThreadState *tstate, PyThreadState **p_save_tstate)
+{
+    PyThreadState *save_tstate = NULL;
+    PyThreadState *cur_tstate = PyThreadState_GET();
+    if (tstate == NULL) {
+        if (PyThreadState_GetInterpreter(cur_tstate) == interp) {
+            tstate = cur_tstate;
+        }
+        else {
+            tstate = PyThreadState_New(interp);
+            _PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
+            assert(tstate != NULL);
+            save_tstate = PyThreadState_Swap(tstate);
+        }
+    }
+    else {
+        assert(PyThreadState_GetInterpreter(tstate) == interp);
+        if (tstate != cur_tstate) {
+            assert(PyThreadState_GetInterpreter(cur_tstate) != interp);
+            save_tstate = PyThreadState_Swap(tstate);
+        }
+    }
+
+    long whence = _PyInterpreterState_GetWhence(interp);
+    assert(whence != _PyInterpreterState_WHENCE_RUNTIME);
+    if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
+        assert(!interp->_ready);
+        PyThreadState *tstate = PyThreadState_New(interp);
+        save_tstate = PyThreadState_Swap(tstate);
+        _PyInterpreterState_Clear(tstate);
+        PyInterpreterState_Delete(interp);
+    }
+    else {
+        Py_EndInterpreter(tstate);
+    }
+
+    if (p_save_tstate != NULL) {
+        save_tstate = *p_save_tstate;
+    }
+    PyThreadState_Swap(save_tstate);
+}
diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h
index 0f324ba..6ecc10c 100644
--- a/Python/crossinterp_exceptions.h
+++ b/Python/crossinterp_exceptions.h
@@ -6,9 +6,9 @@ static PyTypeObject _PyExc_InterpreterError = {
     .tp_name = "interpreters.InterpreterError",
     .tp_doc = PyDoc_STR("A cross-interpreter operation failed"),
     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
-    //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse,
-    //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear,
-    //.tp_base = (PyTypeObject *)PyExc_BaseException,
+    //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
+    //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
+    //.tp_base = (PyTypeObject *)PyExc_Exception,
 };
 PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError;
 
@@ -19,8 +19,8 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = {
     .tp_name = "interpreters.InterpreterNotFoundError",
     .tp_doc = PyDoc_STR("An interpreter was not found"),
     .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
-    //.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse,
-    //.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear,
+    //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
+    //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
     .tp_base = &_PyExc_InterpreterError,
 };
 PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError;
@@ -61,7 +61,7 @@ _get_not_shareable_error_type(PyInterpreterState *interp)
 static int
 init_exceptions(PyInterpreterState *interp)
 {
-    PyTypeObject *base = (PyTypeObject *)PyExc_BaseException;
+    PyTypeObject *base = (PyTypeObject *)PyExc_Exception;
 
     // builtin static types
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 1d315b8..4e83b16 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -477,6 +477,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     if (interp == NULL) {
         return _PyStatus_ERR("can't make main interpreter");
     }
+    assert(interp->_ready);
 
     status = _PyConfig_Write(config, runtime);
     if (_PyStatus_EXCEPTION(status)) {
@@ -631,6 +632,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
     }
     assert(interp != NULL);
     assert(_Py_IsMainInterpreter(interp));
+    _PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME);
+    interp->_ready = 1;
 
     status = _PyConfig_Copy(&interp->config, src_config);
     if (_PyStatus_EXCEPTION(status)) {
@@ -2120,7 +2123,8 @@ Py_Finalize(void)
 */
 
 static PyStatus
-new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
+new_interpreter(PyThreadState **tstate_p,
+                const PyInterpreterConfig *config, long whence)
 {
     PyStatus status;
 
@@ -2143,6 +2147,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
         *tstate_p = NULL;
         return _PyStatus_OK();
     }
+    _PyInterpreterState_SetWhence(interp, whence);
+    interp->_ready = 1;
 
     // XXX Might new_interpreter() have been called without the GIL held?
     PyThreadState *save_tstate = _PyThreadState_GET();
@@ -2231,15 +2237,17 @@ PyStatus
 Py_NewInterpreterFromConfig(PyThreadState **tstate_p,
                             const PyInterpreterConfig *config)
 {
-    return new_interpreter(tstate_p, config);
+    long whence = _PyInterpreterState_WHENCE_CAPI;
+    return new_interpreter(tstate_p, config, whence);
 }
 
 PyThreadState *
 Py_NewInterpreter(void)
 {
     PyThreadState *tstate = NULL;
+    long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI;
     const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
-    PyStatus status = new_interpreter(&tstate, &config);
+    PyStatus status = new_interpreter(&tstate, &config, whence);
     if (_PyStatus_EXCEPTION(status)) {
         Py_ExitStatusException(status);
     }
diff --git a/Python/pystate.c b/Python/pystate.c
index 4a52f64..b0fb210 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -583,6 +583,8 @@ free_interpreter(PyInterpreterState *interp)
     }
 }
 
+static inline int check_interpreter_whence(long);
+
 /* Get the interpreter state to a minimal consistent state.
    Further init happens in pylifecycle.c before it can be used.
    All fields not initialized here are expected to be zeroed out,
@@ -605,12 +607,17 @@ free_interpreter(PyInterpreterState *interp)
 static PyStatus
 init_interpreter(PyInterpreterState *interp,
                  _PyRuntimeState *runtime, int64_t id,
-                 PyInterpreterState *next)
+                 PyInterpreterState *next,
+                 long whence)
 {
     if (interp->_initialized) {
         return _PyStatus_ERR("interpreter already initialized");
     }
 
+    assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET);
+    assert(check_interpreter_whence(whence) == 0);
+    interp->_whence = whence;
+
     assert(runtime != NULL);
     interp->runtime = runtime;
 
@@ -718,8 +725,9 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp)
     }
     interpreters->head = interp;
 
+    long whence = _PyInterpreterState_WHENCE_UNKNOWN;
     status = init_interpreter(interp, runtime,
-                              id, old_head);
+                              id, old_head, whence);
     if (_PyStatus_EXCEPTION(status)) {
         goto error;
     }
@@ -1103,6 +1111,34 @@ _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate)
 // accessors
 //----------
 
+static inline int
+check_interpreter_whence(long whence)
+{
+    if(whence < 0) {
+        return -1;
+    }
+    if (whence > _PyInterpreterState_WHENCE_MAX) {
+        return -1;
+    }
+    return 0;
+}
+
+long
+_PyInterpreterState_GetWhence(PyInterpreterState *interp)
+{
+    assert(check_interpreter_whence(interp->_whence) == 0);
+    return interp->_whence;
+}
+
+void
+_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence)
+{
+    assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET);
+    assert(check_interpreter_whence(whence) == 0);
+    interp->_whence = whence;
+}
+
+
 PyObject *
 PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
 {
@@ -1114,6 +1150,7 @@ PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
     return PyMapping_GetItemString(modules, "__main__");
 }
 
+
 PyObject *
 PyInterpreterState_GetDict(PyInterpreterState *interp)
 {
@@ -1176,6 +1213,20 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
     return interp->id;
 }
 
+PyObject *
+_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
+{
+    if (_PyInterpreterState_IDInitref(interp) != 0) {
+        return NULL;
+    };
+    int64_t interpid = interp->id;
+    if (interpid < 0) {
+        return NULL;
+    }
+    assert(interpid < LLONG_MAX);
+    return PyLong_FromLongLong(interpid);
+}
+
 
 int
 _PyInterpreterState_IDInitref(PyInterpreterState *interp)