diff options
Diffstat (limited to 'Modules/_multiprocessing/multiprocessing.c')
| -rw-r--r-- | Modules/_multiprocessing/multiprocessing.c | 311 |
1 files changed, 222 insertions, 89 deletions
diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 806e638..eecace8 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -3,30 +3,39 @@ * * multiprocessing.c * - * Copyright (c) 2006-2008, R Oudkerk - * Licensed to PSF under a Contributor Agreement. + * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt */ #include "multiprocessing.h" +#if (defined(CMSG_LEN) && defined(SCM_RIGHTS)) + #define HAVE_FD_TRANSFER 1 +#else + #define HAVE_FD_TRANSFER 0 +#endif + +PyObject *create_win32_namespace(void); + +PyObject *pickle_dumps, *pickle_loads, *pickle_protocol; +PyObject *ProcessError, *BufferTooShort; /* * Function which raises exceptions based on error codes */ PyObject * -_PyMp_SetError(PyObject *Type, int num) +mp_SetError(PyObject *Type, int num) { switch (num) { #ifdef MS_WINDOWS case MP_STANDARD_ERROR: if (Type == NULL) - Type = PyExc_OSError; + Type = PyExc_WindowsError; PyErr_SetExcFromWindowsErr(Type, 0); break; case MP_SOCKET_ERROR: if (Type == NULL) - Type = PyExc_OSError; + Type = PyExc_WindowsError; PyErr_SetExcFromWindowsErr(Type, WSAGetLastError()); break; #else /* !MS_WINDOWS */ @@ -40,6 +49,16 @@ _PyMp_SetError(PyObject *Type, int num) case MP_MEMORY_ERROR: PyErr_NoMemory(); break; + case MP_END_OF_FILE: + PyErr_SetNone(PyExc_EOFError); + break; + case MP_EARLY_END_OF_FILE: + PyErr_SetString(PyExc_IOError, + "got end of file during message"); + break; + case MP_BAD_MESSAGE_LENGTH: + PyErr_SetString(PyExc_IOError, "bad message length"); + break; case MP_EXCEPTION_HAS_BEEN_SET: break; default: @@ -49,87 +68,170 @@ _PyMp_SetError(PyObject *Type, int num) return NULL; } + +/* + * Windows only + */ + #ifdef MS_WINDOWS -static PyObject * -multiprocessing_closesocket(PyObject *self, PyObject *args) + +/* On Windows we set an event to signal Ctrl-C; compare with timemodule.c */ + +HANDLE sigint_event = NULL; + +static BOOL WINAPI +ProcessingCtrlHandler(DWORD dwCtrlType) { - HANDLE handle; - int ret; + SetEvent(sigint_event); + return FALSE; +} + +/* + * Unix only + */ + +#else /* !MS_WINDOWS */ + +#if HAVE_FD_TRANSFER - if (!PyArg_ParseTuple(args, F_HANDLE ":closesocket" , &handle)) +/* Functions for transferring file descriptors between processes. + Reimplements some of the functionality of the fdcred + module at http://www.mca-ltd.com/resources/fdcred_1.tgz. */ +/* Based in http://resin.csoft.net/cgi-bin/man.cgi?section=3&topic=CMSG_DATA */ + +static PyObject * +multiprocessing_sendfd(PyObject *self, PyObject *args) +{ + int conn, fd, res; + struct iovec dummy_iov; + char dummy_char; + struct msghdr msg; + struct cmsghdr *cmsg; + union { + struct cmsghdr hdr; + unsigned char buf[CMSG_SPACE(sizeof(int))]; + } cmsgbuf; + + if (!PyArg_ParseTuple(args, "ii", &conn, &fd)) return NULL; + dummy_iov.iov_base = &dummy_char; + dummy_iov.iov_len = 1; + + memset(&msg, 0, sizeof(msg)); + msg.msg_control = &cmsgbuf.buf; + msg.msg_controllen = sizeof(cmsgbuf.buf); + msg.msg_iov = &dummy_iov; + msg.msg_iovlen = 1; + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + * (int *) CMSG_DATA(cmsg) = fd; + Py_BEGIN_ALLOW_THREADS - ret = closesocket((SOCKET) handle); + res = sendmsg(conn, &msg, 0); Py_END_ALLOW_THREADS - if (ret) - return PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); + if (res < 0) + return PyErr_SetFromErrno(PyExc_OSError); Py_RETURN_NONE; } static PyObject * -multiprocessing_recv(PyObject *self, PyObject *args) +multiprocessing_recvfd(PyObject *self, PyObject *args) { - HANDLE handle; - int size, nread; - PyObject *buf; - - if (!PyArg_ParseTuple(args, F_HANDLE "i:recv" , &handle, &size)) + int conn, fd, res; + char dummy_char; + struct iovec dummy_iov; + struct msghdr msg = {0}; + struct cmsghdr *cmsg; + union { + struct cmsghdr hdr; + unsigned char buf[CMSG_SPACE(sizeof(int))]; + } cmsgbuf; + + if (!PyArg_ParseTuple(args, "i", &conn)) return NULL; - buf = PyBytes_FromStringAndSize(NULL, size); - if (!buf) - return NULL; + dummy_iov.iov_base = &dummy_char; + dummy_iov.iov_len = 1; + + memset(&msg, 0, sizeof(msg)); + msg.msg_control = &cmsgbuf.buf; + msg.msg_controllen = sizeof(cmsgbuf.buf); + msg.msg_iov = &dummy_iov; + msg.msg_iovlen = 1; + + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_SPACE(sizeof(int)); + msg.msg_controllen = cmsg->cmsg_len; Py_BEGIN_ALLOW_THREADS - nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0); + res = recvmsg(conn, &msg, 0); Py_END_ALLOW_THREADS - if (nread < 0) { - Py_DECREF(buf); - return PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); + if (res < 0) + return PyErr_SetFromErrno(PyExc_OSError); + + if (msg.msg_controllen < CMSG_LEN(sizeof(int)) || + (cmsg = CMSG_FIRSTHDR(&msg)) == NULL || + cmsg->cmsg_level != SOL_SOCKET || + cmsg->cmsg_type != SCM_RIGHTS || + cmsg->cmsg_len < CMSG_LEN(sizeof(int))) { + /* If at least one control message is present, there should be + no room for any further data in the buffer. */ + PyErr_SetString(PyExc_RuntimeError, "No file descriptor received"); + return NULL; } - _PyBytes_Resize(&buf, nread); - return buf; + + fd = * (int *) CMSG_DATA(cmsg); + return Py_BuildValue("i", fd); } -static PyObject * -multiprocessing_send(PyObject *self, PyObject *args) -{ - HANDLE handle; - Py_buffer buf; - int ret, length; +#endif /* HAVE_FD_TRANSFER */ - if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) - return NULL; +#endif /* !MS_WINDOWS */ - length = (int)Py_MIN(buf.len, INT_MAX); - Py_BEGIN_ALLOW_THREADS - ret = send((SOCKET) handle, buf.buf, length, 0); - Py_END_ALLOW_THREADS +/* + * All platforms + */ + +static PyObject* +multiprocessing_address_of_buffer(PyObject *self, PyObject *obj) +{ + void *buffer; + Py_ssize_t buffer_len; - PyBuffer_Release(&buf); - if (ret < 0) - return PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); - return PyLong_FromLong(ret); + if (PyObject_AsWriteBuffer(obj, &buffer, &buffer_len) < 0) + return NULL; + + return Py_BuildValue("N" F_PY_SSIZE_T, + PyLong_FromVoidPtr(buffer), buffer_len); } -#endif /* * Function table */ static PyMethodDef module_methods[] = { -#ifdef MS_WINDOWS - {"closesocket", multiprocessing_closesocket, METH_VARARGS, ""}, - {"recv", multiprocessing_recv, METH_VARARGS, ""}, - {"send", multiprocessing_send, METH_VARARGS, ""}, -#endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) - {"sem_unlink", _PyMp_sem_unlink, METH_VARARGS, ""}, + {"address_of_buffer", multiprocessing_address_of_buffer, METH_O, + "address_of_buffer(obj) -> int\n" + "Return address of obj assuming obj supports buffer inteface"}, +#if HAVE_FD_TRANSFER + {"sendfd", multiprocessing_sendfd, METH_VARARGS, + "sendfd(sockfd, fd) -> None\n" + "Send file descriptor given by fd over the unix domain socket\n" + "whose file decriptor is sockfd"}, + {"recvfd", multiprocessing_recvfd, METH_VARARGS, + "recvfd(sockfd) -> fd\n" + "Receive a file descriptor over a unix domain socket\n" + "whose file decriptor is sockfd"}, #endif {NULL} }; @@ -139,64 +241,95 @@ static PyMethodDef module_methods[] = { * Initialize */ -static struct PyModuleDef multiprocessing_module = { - PyModuleDef_HEAD_INIT, - "_multiprocessing", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL -}; - - PyMODINIT_FUNC -PyInit__multiprocessing(void) +init_multiprocessing(void) { - PyObject *module, *temp, *value = NULL; + PyObject *module, *temp, *value; /* Initialize module */ - module = PyModule_Create(&multiprocessing_module); + module = Py_InitModule("_multiprocessing", module_methods); if (!module) - return NULL; + return; + + /* Get copy of objects from pickle */ + temp = PyImport_ImportModule(PICKLE_MODULE); + if (!temp) + return; + pickle_dumps = PyObject_GetAttrString(temp, "dumps"); + pickle_loads = PyObject_GetAttrString(temp, "loads"); + pickle_protocol = PyObject_GetAttrString(temp, "HIGHEST_PROTOCOL"); + Py_XDECREF(temp); + + /* Get copy of BufferTooShort */ + temp = PyImport_ImportModule("multiprocessing"); + if (!temp) + return; + BufferTooShort = PyObject_GetAttrString(temp, "BufferTooShort"); + Py_XDECREF(temp); + + /* Add connection type to module */ + if (PyType_Ready(&ConnectionType) < 0) + return; + Py_INCREF(&ConnectionType); + PyModule_AddObject(module, "Connection", (PyObject*)&ConnectionType); #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) - /* Add _PyMp_SemLock type to module */ - if (PyType_Ready(&_PyMp_SemLockType) < 0) - return NULL; - Py_INCREF(&_PyMp_SemLockType); + /* Add SemLock type to module */ + if (PyType_Ready(&SemLockType) < 0) + return; + Py_INCREF(&SemLockType); { PyObject *py_sem_value_max; /* Some systems define SEM_VALUE_MAX as an unsigned value that - * causes it to be negative when used as an int (NetBSD). - * - * Issue #28152: Use (0) instead of 0 to fix a warning on dead code - * when using clang -Wunreachable-code. */ - if ((int)(SEM_VALUE_MAX) < (0)) + * causes it to be negative when used as an int (NetBSD). */ + if ((int)(SEM_VALUE_MAX) < 0) py_sem_value_max = PyLong_FromLong(INT_MAX); else py_sem_value_max = PyLong_FromLong(SEM_VALUE_MAX); if (py_sem_value_max == NULL) - return NULL; - PyDict_SetItemString(_PyMp_SemLockType.tp_dict, "SEM_VALUE_MAX", + return; + PyDict_SetItemString(SemLockType.tp_dict, "SEM_VALUE_MAX", py_sem_value_max); } - PyModule_AddObject(module, "SemLock", (PyObject*)&_PyMp_SemLockType); + PyModule_AddObject(module, "SemLock", (PyObject*)&SemLockType); +#endif + +#ifdef MS_WINDOWS + /* Add PipeConnection to module */ + if (PyType_Ready(&PipeConnectionType) < 0) + return; + Py_INCREF(&PipeConnectionType); + PyModule_AddObject(module, "PipeConnection", + (PyObject*)&PipeConnectionType); + + /* Initialize win32 class and add to multiprocessing */ + temp = create_win32_namespace(); + if (!temp) + return; + PyModule_AddObject(module, "win32", temp); + + /* Initialize the event handle used to signal Ctrl-C */ + sigint_event = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!sigint_event) { + PyErr_SetFromWindowsErr(0); + return; + } + if (!SetConsoleCtrlHandler(ProcessingCtrlHandler, TRUE)) { + PyErr_SetFromWindowsErr(0); + return; + } #endif /* Add configuration macros */ temp = PyDict_New(); if (!temp) - return NULL; - + return; #define ADD_FLAG(name) \ value = Py_BuildValue("i", name); \ - if (value == NULL) { Py_DECREF(temp); return NULL; } \ + if (value == NULL) { Py_DECREF(temp); return; } \ if (PyDict_SetItemString(temp, #name, value) < 0) { \ - Py_DECREF(temp); Py_DECREF(value); return NULL; } \ + Py_DECREF(temp); Py_DECREF(value); return; } \ Py_DECREF(value) #if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) @@ -205,15 +338,15 @@ PyInit__multiprocessing(void) #ifdef HAVE_SEM_TIMEDWAIT ADD_FLAG(HAVE_SEM_TIMEDWAIT); #endif +#ifdef HAVE_FD_TRANSFER + ADD_FLAG(HAVE_FD_TRANSFER); +#endif #ifdef HAVE_BROKEN_SEM_GETVALUE ADD_FLAG(HAVE_BROKEN_SEM_GETVALUE); #endif #ifdef HAVE_BROKEN_SEM_UNLINK ADD_FLAG(HAVE_BROKEN_SEM_UNLINK); #endif - if (PyModule_AddObject(module, "flags", temp) < 0) - return NULL; - - return module; + return; } |