bpo-32604: Improve subinterpreter tests. (#6914)

Add more tests for subinterpreters. This patch also fixes a few small defects in the channel implementation.

1 files changed, 177 insertions, 73 deletions

diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c
index f5e2ea3..5184f65 100644
--- a/Modules/_xxsubinterpretersmodule.c
+++ b/Modules/_xxsubinterpretersmodule.c
@@ -1250,7 +1250,9 @@ _channel_recv(_channels *channels, int64_t id)
     _PyCrossInterpreterData *data = _channel_next(chan, interp->id);
     PyThread_release_lock(mutex);
     if (data == NULL) {
-        PyErr_Format(ChannelEmptyError, "channel %d is empty", id);
+        if (!PyErr_Occurred()) {
+            PyErr_Format(ChannelEmptyError, "channel %d is empty", id);
+        }
         return NULL;
     }
 
@@ -1304,12 +1306,13 @@ typedef struct channelid {
     PyObject_HEAD
     int64_t id;
     int end;
+    int resolve;
     _channels *channels;
 } channelid;
 
 static channelid *
 newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
-             int force)
+             int force, int resolve)
 {
     channelid *self = PyObject_New(channelid, cls);
     if (self == NULL) {
@@ -1317,6 +1320,7 @@ newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels,
     }
     self->id = cid;
     self->end = end;
+    self->resolve = resolve;
     self->channels = channels;
 
     if (_channels_add_id_object(channels, cid) != 0) {
@@ -1337,14 +1341,15 @@ static _channels * _global_channels(void);
 static PyObject *
 channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
 {
-    static char *kwlist[] = {"id", "send", "recv", "force", NULL};
+    static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL};
     PyObject *id;
     int send = -1;
     int recv = -1;
     int force = 0;
+    int resolve = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$ppp:ChannelID.__init__", kwlist,
-                                     &id, &send, &recv, &force))
+                                     "O|$pppp:ChannelID.__new__", kwlist,
+                                     &id, &send, &recv, &force, &resolve))
         return NULL;
 
     // Coerce and check the ID.
@@ -1376,7 +1381,8 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
         end = CHANNEL_RECV;
     }
 
-    return (PyObject *)newchannelid(cls, cid, end, _global_channels(), force);
+    return (PyObject *)newchannelid(cls, cid, end, _global_channels(),
+                                    force, resolve);
 }
 
 static void
@@ -1409,6 +1415,13 @@ channelid_repr(PyObject *self)
     return PyUnicode_FromFormat(fmt, name, cid->id);
 }
 
+static PyObject *
+channelid_str(PyObject *self)
+{
+    channelid *cid = (channelid *)self;
+    return PyUnicode_FromFormat("%d", cid->id);
+}
+
 PyObject *
 channelid_int(PyObject *self)
 {
@@ -1519,14 +1532,49 @@ channelid_richcompare(PyObject *self, PyObject *other, int op)
 struct _channelid_xid {
     int64_t id;
     int end;
+    int resolve;
 };
 
 static PyObject *
 _channelid_from_xid(_PyCrossInterpreterData *data)
 {
     struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
-    return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end,
-                                    _global_channels(), 0);
+    // Note that we do not preserve the "resolve" flag.
+    PyObject *cid = (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end,
+                                             _global_channels(), 0, 0);
+    if (xid->end == 0) {
+        return cid;
+    }
+    if (!xid->resolve) {
+        return cid;
+    }
+
+    /* Try returning a high-level channel end but fall back to the ID. */
+    PyObject *highlevel = PyImport_ImportModule("interpreters");
+    if (highlevel == NULL) {
+        PyErr_Clear();
+        highlevel = PyImport_ImportModule("test.support.interpreters");
+        if (highlevel == NULL) {
+            goto error;
+        }
+    }
+    const char *clsname = (xid->end == CHANNEL_RECV) ? "RecvChannel" :
+                                                       "SendChannel";
+    PyObject *cls = PyObject_GetAttrString(highlevel, clsname);
+    Py_DECREF(highlevel);
+    if (cls == NULL) {
+        goto error;
+    }
+    PyObject *chan = PyObject_CallFunctionObjArgs(cls, cid, NULL);
+    if (chan == NULL) {
+        goto error;
+    }
+    Py_DECREF(cid);
+    return chan;
+
+error:
+    PyErr_Clear();
+    return cid;
 }
 
 static int
@@ -1538,6 +1586,7 @@ _channelid_shared(PyObject *obj, _PyCrossInterpreterData *data)
     }
     xid->id = ((channelid *)obj)->id;
     xid->end = ((channelid *)obj)->end;
+    xid->resolve = ((channelid *)obj)->resolve;
 
     data->data = xid;
     data->obj = obj;
@@ -1553,7 +1602,7 @@ channelid_end(PyObject *self, void *end)
     channelid *cid = (channelid *)self;
     if (end != NULL) {
         return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end,
-                                        cid->channels, force);
+                                        cid->channels, force, cid->resolve);
     }
 
     if (cid->end == CHANNEL_SEND) {
@@ -1597,7 +1646,7 @@ static PyTypeObject ChannelIDtype = {
     0,                              /* tp_as_mapping */
     channelid_hash,                 /* tp_hash */
     0,                              /* tp_call */
-    0,                              /* tp_str */
+    (reprfunc)channelid_str,        /* tp_str */
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
@@ -1878,6 +1927,13 @@ interpid_repr(PyObject *self)
     return PyUnicode_FromFormat("%s(%d)", name, id->id);
 }
 
+static PyObject *
+interpid_str(PyObject *self)
+{
+    interpid *id = (interpid *)self;
+    return PyUnicode_FromFormat("%d", id->id);
+}
+
 PyObject *
 interpid_int(PyObject *self)
 {
@@ -1999,7 +2055,7 @@ static PyTypeObject InterpreterIDtype = {
     0,                              /* tp_as_mapping */
     interpid_hash,                  /* tp_hash */
     0,                              /* tp_call */
-    0,                              /* tp_str */
+    (reprfunc)interpid_str,         /* tp_str */
     0,                              /* tp_getattro */
     0,                              /* tp_setattro */
     0,                              /* tp_as_buffer */
@@ -2115,10 +2171,13 @@ Create a new interpreter and return a unique generated ID.");
 
 
 static PyObject *
-interp_destroy(PyObject *self, PyObject *args)
+interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"id", NULL};
     PyObject *id;
-    if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) {
+    // XXX Use "L" for id?
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O:destroy", kwlist, &id)) {
         return NULL;
     }
     if (!PyLong_Check(id)) {
@@ -2162,7 +2221,7 @@ interp_destroy(PyObject *self, PyObject *args)
 }
 
 PyDoc_STRVAR(destroy_doc,
-"destroy(ID)\n\
+"destroy(id)\n\
 \n\
 Destroy the identified interpreter.\n\
 \n\
@@ -2228,7 +2287,8 @@ static PyObject *
 interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     // Currently, 0 is always the main interpreter.
-    return PyLong_FromLongLong(0);
+    PY_INT64_T id = 0;
+    return (PyObject *)newinterpid(&InterpreterIDtype, id, 0);
 }
 
 PyDoc_STRVAR(get_main_doc,
@@ -2238,22 +2298,20 @@ Return the ID of main interpreter.");
 
 
 static PyObject *
-interp_run_string(PyObject *self, PyObject *args)
+interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"id", "script", "shared", NULL};
     PyObject *id, *code;
     PyObject *shared = NULL;
-    if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "OU|O:run_string", kwlist,
+                                     &id, &code, &shared)) {
         return NULL;
     }
     if (!PyLong_Check(id)) {
         PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int");
         return NULL;
     }
-    if (!PyUnicode_Check(code)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "second arg (code) must be a string");
-        return NULL;
-    }
 
     // Look up the interpreter.
     PyInterpreterState *interp = _look_up(id);
@@ -2281,7 +2339,7 @@ interp_run_string(PyObject *self, PyObject *args)
 }
 
 PyDoc_STRVAR(run_string_doc,
-"run_string(ID, sourcetext)\n\
+"run_string(id, script, shared)\n\
 \n\
 Execute the provided string in the identified interpreter.\n\
 \n\
@@ -2289,12 +2347,15 @@ See PyRun_SimpleStrings.");
 
 
 static PyObject *
-object_is_shareable(PyObject *self, PyObject *args)
+object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"obj", NULL};
     PyObject *obj;
-    if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O:is_shareable", kwlist, &obj)) {
         return NULL;
     }
+
     if (_PyObject_CheckCrossInterpreterData(obj) == 0) {
         Py_RETURN_TRUE;
     }
@@ -2310,10 +2371,12 @@ False otherwise.");
 
 
 static PyObject *
-interp_is_running(PyObject *self, PyObject *args)
+interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"id", NULL};
     PyObject *id;
-    if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O:is_running", kwlist, &id)) {
         return NULL;
     }
     if (!PyLong_Check(id)) {
@@ -2348,7 +2411,7 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored))
         return NULL;
     }
     PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0,
-                                            &_globals.channels, 0);
+                                            &_globals.channels, 0, 0);
     if (id == NULL) {
         if (_channel_destroy(&_globals.channels, cid) != 0) {
             // XXX issue a warning?
@@ -2360,15 +2423,17 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 
 PyDoc_STRVAR(channel_create_doc,
-"channel_create() -> ID\n\
+"channel_create() -> cid\n\
 \n\
 Create a new cross-interpreter channel and return a unique generated ID.");
 
 static PyObject *
-channel_destroy(PyObject *self, PyObject *args)
+channel_destroy(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"cid", NULL};
     PyObject *id;
-    if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O:channel_destroy", kwlist, &id)) {
         return NULL;
     }
     int64_t cid = _coerce_id(id);
@@ -2383,7 +2448,7 @@ channel_destroy(PyObject *self, PyObject *args)
 }
 
 PyDoc_STRVAR(channel_destroy_doc,
-"channel_destroy(ID)\n\
+"channel_destroy(cid)\n\
 \n\
 Close and finalize the channel.  Afterward attempts to use the channel\n\
 will behave as though it never existed.");
@@ -2406,7 +2471,7 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
     int64_t *cur = cids;
     for (int64_t i=0; i < count; cur++, i++) {
         PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cur, 0,
-                                                &_globals.channels, 0);
+                                                &_globals.channels, 0, 0);
         if (id == NULL) {
             Py_DECREF(ids);
             ids = NULL;
@@ -2421,16 +2486,18 @@ finally:
 }
 
 PyDoc_STRVAR(channel_list_all_doc,
-"channel_list_all() -> [ID]\n\
+"channel_list_all() -> [cid]\n\
 \n\
 Return the list of all IDs for active channels.");
 
 static PyObject *
-channel_send(PyObject *self, PyObject *args)
+channel_send(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"cid", "obj", NULL};
     PyObject *id;
     PyObject *obj;
-    if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "OO:channel_send", kwlist, &id, &obj)) {
         return NULL;
     }
     int64_t cid = _coerce_id(id);
@@ -2445,15 +2512,17 @@ channel_send(PyObject *self, PyObject *args)
 }
 
 PyDoc_STRVAR(channel_send_doc,
-"channel_send(ID, obj)\n\
+"channel_send(cid, obj)\n\
 \n\
 Add the object's data to the channel's queue.");
 
 static PyObject *
-channel_recv(PyObject *self, PyObject *args)
+channel_recv(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"cid", NULL};
     PyObject *id;
-    if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O:channel_recv", kwlist, &id)) {
         return NULL;
     }
     int64_t cid = _coerce_id(id);
@@ -2465,17 +2534,34 @@ channel_recv(PyObject *self, PyObject *args)
 }
 
 PyDoc_STRVAR(channel_recv_doc,
-"channel_recv(ID) -> obj\n\
+"channel_recv(cid) -> obj\n\
 \n\
 Return a new object from the data at the from of the channel's queue.");
 
 static PyObject *
-channel_close(PyObject *self, PyObject *id)
+channel_close(PyObject *self, PyObject *args, PyObject *kwds)
 {
+    static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
+    PyObject *id;
+    int send = 0;
+    int recv = 0;
+    int force = 0;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O|$ppp:channel_close", kwlist,
+                                     &id, &send, &recv, &force)) {
+        return NULL;
+    }
     int64_t cid = _coerce_id(id);
     if (cid < 0) {
         return NULL;
     }
+    if (send == 0 && recv == 0) {
+        send = 1;
+        recv = 1;
+    }
+
+    // XXX Handle the ends.
+    // XXX Handle force is True.
 
     if (_channel_close(&_globals.channels, cid) != 0) {
         return NULL;
@@ -2484,48 +2570,66 @@ channel_close(PyObject *self, PyObject *id)
 }
 
 PyDoc_STRVAR(channel_close_doc,
-"channel_close(ID)\n\
+"channel_close(cid, *, send=None, recv=None, force=False)\n\
+\n\
+Close the channel for all interpreters.\n\
 \n\
-Close the channel for all interpreters.  Once the channel's ID has\n\
-no more ref counts the channel will be destroyed.");
+If the channel is empty then the keyword args are ignored and both\n\
+ends are immediately closed.  Otherwise, if 'force' is True then\n\
+all queued items are released and both ends are immediately\n\
+closed.\n\
+\n\
+If the channel is not empty *and* 'force' is False then following\n\
+happens:\n\
+\n\
+ * recv is True (regardless of send):\n\
+   - raise ChannelNotEmptyError\n\
+ * recv is None and send is None:\n\
+   - raise ChannelNotEmptyError\n\
+ * send is True and recv is not True:\n\
+   - fully close the 'send' end\n\
+   - close the 'recv' end to interpreters not already receiving\n\
+   - fully close it once empty\n\
+\n\
+Closing an already closed channel results in a ChannelClosedError.\n\
+\n\
+Once the channel's ID has no more ref counts in any interpreter\n\
+the channel will be destroyed.");
 
 static PyObject *
-channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds)
+channel_release(PyObject *self, PyObject *args, PyObject *kwds)
 {
     // Note that only the current interpreter is affected.
-    static char *kwlist[] = {"id", "send", "recv", NULL};
+    static char *kwlist[] = {"cid", "send", "recv", "force", NULL};
     PyObject *id;
-    int send = -1;
-    int recv = -1;
+    int send = 0;
+    int recv = 0;
+    int force = 0;
     if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$pp:channel_drop_interpreter", kwlist,
-                                     &id, &send, &recv))
+                                     "O|$ppp:channel_release", kwlist,
+                                     &id, &send, &recv, &force)) {
         return NULL;
-
+    }
     int64_t cid = _coerce_id(id);
     if (cid < 0) {
         return NULL;
     }
-    if (send < 0 && recv < 0) {
+    if (send == 0 && recv == 0) {
         send = 1;
         recv = 1;
     }
-    else {
-        if (send < 0) {
-            send = 0;
-        }
-        if (recv < 0) {
-            recv = 0;
-        }
-    }
+
+    // XXX Handle force is True.
+    // XXX Fix implicit release.
+
     if (_channel_drop(&_globals.channels, cid, send, recv) != 0) {
         return NULL;
     }
     Py_RETURN_NONE;
 }
 
-PyDoc_STRVAR(channel_drop_interpreter_doc,
-"channel_drop_interpreter(ID, *, send=None, recv=None)\n\
+PyDoc_STRVAR(channel_release_doc,
+"channel_release(cid, *, send=None, recv=None, force=True)\n\
 \n\
 Close the channel for the current interpreter.  'send' and 'recv'\n\
 (bool) may be used to indicate the ends to close.  By default both\n\
@@ -2541,7 +2645,7 @@ static PyMethodDef module_functions[] = {
     {"create",                    (PyCFunction)interp_create,
      METH_VARARGS, create_doc},
     {"destroy",                   (PyCFunction)interp_destroy,
-     METH_VARARGS, destroy_doc},
+     METH_VARARGS | METH_KEYWORDS, destroy_doc},
     {"list_all",                  interp_list_all,
      METH_NOARGS, list_all_doc},
     {"get_current",               interp_get_current,
@@ -2549,27 +2653,27 @@ static PyMethodDef module_functions[] = {
     {"get_main",                  interp_get_main,
      METH_NOARGS, get_main_doc},
     {"is_running",                (PyCFunction)interp_is_running,
-     METH_VARARGS, is_running_doc},
+     METH_VARARGS | METH_KEYWORDS, is_running_doc},
     {"run_string",                (PyCFunction)interp_run_string,
-     METH_VARARGS, run_string_doc},
+     METH_VARARGS | METH_KEYWORDS, run_string_doc},
 
     {"is_shareable",              (PyCFunction)object_is_shareable,
-     METH_VARARGS, is_shareable_doc},
+     METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
 
     {"channel_create",            channel_create,
      METH_NOARGS, channel_create_doc},
     {"channel_destroy",           (PyCFunction)channel_destroy,
-     METH_VARARGS, channel_destroy_doc},
+     METH_VARARGS | METH_KEYWORDS, channel_destroy_doc},
     {"channel_list_all",          channel_list_all,
      METH_NOARGS, channel_list_all_doc},
     {"channel_send",              (PyCFunction)channel_send,
-     METH_VARARGS, channel_send_doc},
+     METH_VARARGS | METH_KEYWORDS, channel_send_doc},
     {"channel_recv",              (PyCFunction)channel_recv,
-     METH_VARARGS, channel_recv_doc},
-    {"channel_close",             channel_close,
-     METH_O, channel_close_doc},
-    {"channel_drop_interpreter",  (PyCFunction)channel_drop_interpreter,
-     METH_VARARGS | METH_KEYWORDS, channel_drop_interpreter_doc},
+     METH_VARARGS | METH_KEYWORDS, channel_recv_doc},
+    {"channel_close",             (PyCFunction)channel_close,
+     METH_VARARGS | METH_KEYWORDS, channel_close_doc},
+    {"channel_release",           (PyCFunction)channel_release,
+     METH_VARARGS | METH_KEYWORDS, channel_release_doc},
     {"_channel_id",               (PyCFunction)channel__channel_id,
      METH_VARARGS | METH_KEYWORDS, NULL},