diff options
Diffstat (limited to 'Modules/_posixsubprocess.c')
| -rw-r--r-- | Modules/_posixsubprocess.c | 338 |
1 files changed, 145 insertions, 193 deletions
diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 0196ca2..1490223 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -14,10 +14,19 @@ #ifdef HAVE_SYS_SYSCALL_H #include <sys/syscall.h> #endif +#if defined(HAVE_SYS_RESOURCE_H) +#include <sys/resource.h> +#endif #ifdef HAVE_DIRENT_H #include <dirent.h> #endif +#if defined(__ANDROID__) && !defined(SYS_getdents64) +/* Android doesn't expose syscalls, add the definition manually. */ +# include <sys/linux-syscalls.h> +# define SYS_getdents64 __NR_getdents64 +#endif + #if defined(sun) /* readdir64 is used to work around Solaris 9 bug 6395699. */ # define readdir readdir64 @@ -35,24 +44,28 @@ # define FD_DIR "/proc/self/fd" #endif -#define POSIX_CALL(call) if ((call) == -1) goto error +#define POSIX_CALL(call) do { if ((call) == -1) goto error; } while (0) -/* Maximum file descriptor, initialized on module load. */ -static long max_fd; - - -/* Given the gc module call gc.enable() and return 0 on success. */ +/* If gc was disabled, call gc.enable(). Return 0 on success. */ static int -_enable_gc(PyObject *gc_module) +_enable_gc(int need_to_reenable_gc, PyObject *gc_module) { PyObject *result; _Py_IDENTIFIER(enable); + PyObject *exctype, *val, *tb; - result = _PyObject_CallMethodId(gc_module, &PyId_enable, NULL); - if (result == NULL) - return 1; - Py_DECREF(result); + if (need_to_reenable_gc) { + PyErr_Fetch(&exctype, &val, &tb); + result = _PyObject_CallMethodId(gc_module, &PyId_enable, NULL); + if (exctype != NULL) { + PyErr_Restore(exctype, val, tb); + } + if (result == NULL) { + return 1; + } + Py_DECREF(result); + } return 0; } @@ -87,7 +100,7 @@ _is_fdescfs_mounted_on_dev_fd(void) if (stat("/dev", &dev_stat) != 0) return 0; if (stat(FD_DIR, &dev_fd_stat) != 0) - return 0; + return 0; if (dev_stat.st_dev == dev_fd_stat.st_dev) return 0; /* / == /dev == /dev/fd means it is static. #fail */ return 1; @@ -136,15 +149,70 @@ _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) return 0; } +static int +make_inheritable(PyObject *py_fds_to_keep, int errpipe_write) +{ + Py_ssize_t i, len; + + len = PySequence_Length(py_fds_to_keep); + for (i = 0; i < len; ++i) { + PyObject* fdobj = PySequence_Fast_GET_ITEM(py_fds_to_keep, i); + long fd = PyLong_AsLong(fdobj); + assert(!PyErr_Occurred()); + assert(0 <= fd && fd <= INT_MAX); + if (fd == errpipe_write) { + /* errpipe_write is part of py_fds_to_keep. It must be closed at + exec(), but kept open in the child process until exec() is + called. */ + continue; + } + if (_Py_set_inheritable((int)fd, 1, NULL) < 0) + return -1; + } + return 0; +} + -/* Close all file descriptors in the range start_fd inclusive to - * end_fd exclusive except for those in py_fds_to_keep. If the - * range defined by [start_fd, end_fd) is large this will take a - * long time as it calls close() on EVERY possible fd. +/* Get the maximum file descriptor that could be opened by this process. + * This function is async signal safe for use between fork() and exec(). + */ +static long +safe_get_max_fd(void) +{ + long local_max_fd; +#if defined(__NetBSD__) + local_max_fd = fcntl(0, F_MAXFD); + if (local_max_fd >= 0) + return local_max_fd; +#endif +#if defined(HAVE_SYS_RESOURCE_H) && defined(__OpenBSD__) + struct rlimit rl; + /* Not on the POSIX async signal safe functions list but likely + * safe. TODO - Someone should audit OpenBSD to make sure. */ + if (getrlimit(RLIMIT_NOFILE, &rl) >= 0) + return (long) rl.rlim_max; +#endif +#ifdef _SC_OPEN_MAX + local_max_fd = sysconf(_SC_OPEN_MAX); + if (local_max_fd == -1) +#endif + local_max_fd = 256; /* Matches legacy Lib/subprocess.py behavior. */ + return local_max_fd; +} + + +/* Close all file descriptors in the range from start_fd and higher + * except for those in py_fds_to_keep. If the range defined by + * [start_fd, safe_get_max_fd()) is large this will take a long + * time as it calls close() on EVERY possible fd. + * + * It isn't possible to know for sure what the max fd to go up to + * is for processes with the capability of raising their maximum. */ static void -_close_fds_by_brute_force(int start_fd, int end_fd, PyObject *py_fds_to_keep) +_close_fds_by_brute_force(long start_fd, PyObject *py_fds_to_keep) { + long end_fd = safe_get_max_fd(); Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep); Py_ssize_t keep_seq_idx; int fd_num; @@ -157,13 +225,13 @@ _close_fds_by_brute_force(int start_fd, int end_fd, PyObject *py_fds_to_keep) if (keep_fd < start_fd) continue; for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) { - while (close(fd_num) < 0 && errno == EINTR); + close(fd_num); } start_fd = keep_fd + 1; } if (start_fd <= end_fd) { for (fd_num = start_fd; fd_num < end_fd; ++fd_num) { - while (close(fd_num) < 0 && errno == EINTR); + close(fd_num); } } } @@ -184,8 +252,8 @@ struct linux_dirent64 { char d_name[256]; /* Filename (null-terminated) */ }; -/* Close all open file descriptors in the range start_fd inclusive to end_fd - * exclusive. Do not close any in the sorted py_fds_to_keep list. +/* Close all open file descriptors in the range from start_fd and higher + * Do not close any in the sorted py_fds_to_keep list. * * This version is async signal safe as it does not make any unsafe C library * calls, malloc calls or handle any locks. It is _unfortunate_ to be forced @@ -200,26 +268,14 @@ struct linux_dirent64 { * it with some cpp #define magic to work on other OSes as well if you want. */ static void -_close_open_fd_range_safe(int start_fd, int end_fd, PyObject* py_fds_to_keep) +_close_open_fds_safe(int start_fd, PyObject* py_fds_to_keep) { int fd_dir_fd; - if (start_fd >= end_fd) - return; -#ifdef O_CLOEXEC - fd_dir_fd = open(FD_DIR, O_RDONLY | O_CLOEXEC, 0); -#else - fd_dir_fd = open(FD_DIR, O_RDONLY, 0); -#ifdef FD_CLOEXEC - { - int old = fcntl(fd_dir_fd, F_GETFD); - if (old != -1) - fcntl(fd_dir_fd, F_SETFD, old | FD_CLOEXEC); - } -#endif -#endif + + fd_dir_fd = _Py_open(FD_DIR, O_RDONLY); if (fd_dir_fd == -1) { /* No way to get a list of open fds. */ - _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep); + _close_fds_by_brute_force(start_fd, py_fds_to_keep); return; } else { char buffer[sizeof(struct linux_dirent64)]; @@ -234,9 +290,9 @@ _close_open_fd_range_safe(int start_fd, int end_fd, PyObject* py_fds_to_keep) entry = (struct linux_dirent64 *)(buffer + offset); if ((fd = _pos_int_from_ascii(entry->d_name)) < 0) continue; /* Not a number. */ - if (fd != fd_dir_fd && fd >= start_fd && fd < end_fd && + if (fd != fd_dir_fd && fd >= start_fd && !_is_fd_in_sorted_fd_sequence(fd, py_fds_to_keep)) { - while (close(fd) < 0 && errno == EINTR); + close(fd); } } } @@ -244,13 +300,13 @@ _close_open_fd_range_safe(int start_fd, int end_fd, PyObject* py_fds_to_keep) } } -#define _close_open_fd_range _close_open_fd_range_safe +#define _close_open_fds _close_open_fds_safe #else /* NOT (defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)) */ -/* Close all open file descriptors in the range start_fd inclusive to end_fd - * exclusive. Do not close any in the sorted py_fds_to_keep list. +/* Close all open file descriptors from start_fd and higher. + * Do not close any in the sorted py_fds_to_keep list. * * This function violates the strict use of async signal safe functions. :( * It calls opendir(), readdir() and closedir(). Of these, the one most @@ -263,26 +319,20 @@ _close_open_fd_range_safe(int start_fd, int end_fd, PyObject* py_fds_to_keep) * http://womble.decadent.org.uk/readdir_r-advisory.html */ static void -_close_open_fd_range_maybe_unsafe(int start_fd, int end_fd, - PyObject* py_fds_to_keep) +_close_open_fds_maybe_unsafe(long start_fd, PyObject* py_fds_to_keep) { DIR *proc_fd_dir; #ifndef HAVE_DIRFD - while (_is_fd_in_sorted_fd_sequence(start_fd, py_fds_to_keep) && - (start_fd < end_fd)) { + while (_is_fd_in_sorted_fd_sequence(start_fd, py_fds_to_keep)) { ++start_fd; } - if (start_fd >= end_fd) - return; /* Close our lowest fd before we call opendir so that it is likely to * reuse that fd otherwise we might close opendir's file descriptor in * our loop. This trick assumes that fd's are allocated on a lowest * available basis. */ - while (close(start_fd) < 0 && errno == EINTR); + close(start_fd); ++start_fd; #endif - if (start_fd >= end_fd) - return; #if defined(__FreeBSD__) if (!_is_fdescfs_mounted_on_dev_fd()) @@ -292,7 +342,7 @@ _close_open_fd_range_maybe_unsafe(int start_fd, int end_fd, proc_fd_dir = opendir(FD_DIR); if (!proc_fd_dir) { /* No way to get a list of open fds. */ - _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep); + _close_fds_by_brute_force(start_fd, py_fds_to_keep); } else { struct dirent *dir_entry; #ifdef HAVE_DIRFD @@ -305,21 +355,21 @@ _close_open_fd_range_maybe_unsafe(int start_fd, int end_fd, int fd; if ((fd = _pos_int_from_ascii(dir_entry->d_name)) < 0) continue; /* Not a number. */ - if (fd != fd_used_by_opendir && fd >= start_fd && fd < end_fd && + if (fd != fd_used_by_opendir && fd >= start_fd && !_is_fd_in_sorted_fd_sequence(fd, py_fds_to_keep)) { - while (close(fd) < 0 && errno == EINTR); + close(fd); } errno = 0; } if (errno) { /* readdir error, revert behavior. Highly Unlikely. */ - _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep); + _close_fds_by_brute_force(start_fd, py_fds_to_keep); } closedir(proc_fd_dir); } } -#define _close_open_fd_range _close_open_fd_range_maybe_unsafe +#define _close_open_fds _close_open_fds_maybe_unsafe #endif /* else NOT (defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)) */ @@ -356,16 +406,16 @@ child_exec(char *const exec_array[], /* Buffer large enough to hold a hex integer. We can't malloc. */ char hex_errno[sizeof(saved_errno)*2+1]; + if (make_inheritable(py_fds_to_keep, errpipe_write) < 0) + goto error; + /* Close parent's pipe ends. */ - if (p2cwrite != -1) { + if (p2cwrite != -1) POSIX_CALL(close(p2cwrite)); - } - if (c2pread != -1) { + if (c2pread != -1) POSIX_CALL(close(c2pread)); - } - if (errread != -1) { + if (errread != -1) POSIX_CALL(close(errread)); - } POSIX_CALL(close(errpipe_read)); /* When duping fds, if there arises a situation where one of the fds is @@ -379,38 +429,34 @@ child_exec(char *const exec_array[], dup2() removes the CLOEXEC flag but we must do it ourselves if dup2() would be a no-op (issue #10806). */ if (p2cread == 0) { - int old = fcntl(p2cread, F_GETFD); - if (old != -1) - fcntl(p2cread, F_SETFD, old & ~FD_CLOEXEC); - } else if (p2cread != -1) { - POSIX_CALL(dup2(p2cread, 0)); /* stdin */ + if (_Py_set_inheritable(p2cread, 1, NULL) < 0) + goto error; } + else if (p2cread != -1) + POSIX_CALL(dup2(p2cread, 0)); /* stdin */ + if (c2pwrite == 1) { - int old = fcntl(c2pwrite, F_GETFD); - if (old != -1) - fcntl(c2pwrite, F_SETFD, old & ~FD_CLOEXEC); - } else if (c2pwrite != -1) { - POSIX_CALL(dup2(c2pwrite, 1)); /* stdout */ + if (_Py_set_inheritable(c2pwrite, 1, NULL) < 0) + goto error; } + else if (c2pwrite != -1) + POSIX_CALL(dup2(c2pwrite, 1)); /* stdout */ + if (errwrite == 2) { - int old = fcntl(errwrite, F_GETFD); - if (old != -1) - fcntl(errwrite, F_SETFD, old & ~FD_CLOEXEC); - } else if (errwrite != -1) { - POSIX_CALL(dup2(errwrite, 2)); /* stderr */ + if (_Py_set_inheritable(errwrite, 1, NULL) < 0) + goto error; } + else if (errwrite != -1) + POSIX_CALL(dup2(errwrite, 2)); /* stderr */ /* Close pipe fds. Make sure we don't close the same fd more than */ /* once, or standard fds. */ - if (p2cread > 2) { + if (p2cread > 2) POSIX_CALL(close(p2cread)); - } - if (c2pwrite > 2 && c2pwrite != p2cread) { + if (c2pwrite > 2 && c2pwrite != p2cread) POSIX_CALL(close(c2pwrite)); - } - if (errwrite != c2pwrite && errwrite != p2cread && errwrite > 2) { + if (errwrite != c2pwrite && errwrite != p2cread && errwrite > 2) POSIX_CALL(close(errwrite)); - } if (cwd) POSIX_CALL(chdir(cwd)); @@ -442,14 +488,8 @@ child_exec(char *const exec_array[], /* close FDs after executing preexec_fn, which might open FDs */ if (close_fds) { - int local_max_fd = max_fd; -#if defined(__NetBSD__) - local_max_fd = fcntl(0, F_MAXFD); - if (local_max_fd < 0) - local_max_fd = max_fd; -#endif /* TODO HP-UX could use pstat_getproc() if anyone cares about it. */ - _close_open_fd_range(3, local_max_fd, py_fds_to_keep); + _close_open_fds(3, py_fds_to_keep); } /* This loop matches the Lib/os.py _execvpe()'s PATH search when */ @@ -492,7 +532,7 @@ error: /* We can't call strerror(saved_errno). It is not async signal safe. * The parent process will look the error message up. */ } else { - unused = write(errpipe_write, "RuntimeError:0:", 15); + unused = write(errpipe_write, "SubprocessError:0:", 18); unused = write(errpipe_write, err_msg, strlen(err_msg)); } if (unused) return; /* silly? yes! avoids gcc compiler warning. */ @@ -516,6 +556,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) int need_to_reenable_gc = 0; char *const *exec_array, *const *argv = NULL, *const *envp = NULL; Py_ssize_t arg_num; + int import_lock_held = 0; if (!PyArg_ParseTuple( args, "OOpOOOiiiiiiiiiiO:fork_exec", @@ -544,7 +585,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) PyObject *result; _Py_IDENTIFIER(isenabled); _Py_IDENTIFIER(disable); - + gc_module = PyImport_ImportModule("gc"); if (gc_module == NULL) return NULL; @@ -568,10 +609,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args) } exec_array = _PySequence_BytesToCharpArray(executable_list); - if (!exec_array) { - Py_XDECREF(gc_module); - return NULL; - } + if (!exec_array) + goto cleanup; /* Convert args and env into appropriate arguments for exec() */ /* These conversions are done in the parent process to avoid allocating @@ -613,6 +652,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) if (!preexec_fn_args_tuple) goto cleanup; _PyImport_AcquireLock(); + import_lock_held = 1; } if (cwd_obj != Py_None) { @@ -659,7 +699,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); + pid = -1; } + import_lock_held = 0; /* Parent process */ if (envp) @@ -669,9 +711,8 @@ subprocess_fork_exec(PyObject* self, PyObject *args) _Py_FreeCharPArray(exec_array); /* Reenable gc in the parent process (or if fork failed). */ - if (need_to_reenable_gc && _enable_gc(gc_module)) { - Py_XDECREF(gc_module); - return NULL; + if (_enable_gc(need_to_reenable_gc, gc_module)) { + pid = -1; } Py_XDECREF(preexec_fn_args_tuple); Py_XDECREF(gc_module); @@ -682,18 +723,18 @@ subprocess_fork_exec(PyObject* self, PyObject *args) return PyLong_FromPid(pid); cleanup: + if (import_lock_held) + _PyImport_ReleaseLock(); if (envp) _Py_FreeCharPArray(envp); if (argv) _Py_FreeCharPArray(argv); - _Py_FreeCharPArray(exec_array); + if (exec_array) + _Py_FreeCharPArray(exec_array); Py_XDECREF(converted_args); Py_XDECREF(fast_args); Py_XDECREF(preexec_fn_args_tuple); - - /* Reenable gc if it was disabled. */ - if (need_to_reenable_gc) - _enable_gc(gc_module); + _enable_gc(need_to_reenable_gc, gc_module); Py_XDECREF(gc_module); return NULL; } @@ -721,88 +762,6 @@ Returns: the child process's PID.\n\ Raises: Only on an error in the parent process.\n\ "); -PyDoc_STRVAR(subprocess_cloexec_pipe_doc, -"cloexec_pipe() -> (read_end, write_end)\n\n\ -Create a pipe whose ends have the cloexec flag set; write_end will be >= 3."); - -static PyObject * -subprocess_cloexec_pipe(PyObject *self, PyObject *noargs) -{ - int fds[2]; - int res, saved_errno; - long oldflags; -#ifdef HAVE_PIPE2 - Py_BEGIN_ALLOW_THREADS - res = pipe2(fds, O_CLOEXEC); - Py_END_ALLOW_THREADS - if (res != 0 && errno == ENOSYS) - { -#endif - /* We hold the GIL which offers some protection from other code calling - * fork() before the CLOEXEC flags have been set but we can't guarantee - * anything without pipe2(). */ - res = pipe(fds); - - if (res == 0) { - oldflags = fcntl(fds[0], F_GETFD, 0); - if (oldflags < 0) res = oldflags; - } - if (res == 0) - res = fcntl(fds[0], F_SETFD, oldflags | FD_CLOEXEC); - - if (res == 0) { - oldflags = fcntl(fds[1], F_GETFD, 0); - if (oldflags < 0) res = oldflags; - } - if (res == 0) - res = fcntl(fds[1], F_SETFD, oldflags | FD_CLOEXEC); -#ifdef HAVE_PIPE2 - } -#endif - if (res == 0 && fds[1] < 3) { - /* We always want the write end of the pipe to avoid fds 0, 1 and 2 - * as our child may claim those for stdio connections. */ - int write_fd = fds[1]; - int fds_to_close[3] = {-1, -1, -1}; - int fds_to_close_idx = 0; -#ifdef F_DUPFD_CLOEXEC - fds_to_close[fds_to_close_idx++] = write_fd; - write_fd = fcntl(write_fd, F_DUPFD_CLOEXEC, 3); - if (write_fd < 0) /* We don't support F_DUPFD_CLOEXEC / other error */ -#endif - { - /* Use dup a few times until we get a desirable fd. */ - for (; fds_to_close_idx < 3; ++fds_to_close_idx) { - fds_to_close[fds_to_close_idx] = write_fd; - write_fd = dup(write_fd); - if (write_fd >= 3) - break; - /* We may dup a few extra times if it returns an error but - * that is okay. Repeat calls should return the same error. */ - } - if (write_fd < 0) res = write_fd; - if (res == 0) { - oldflags = fcntl(write_fd, F_GETFD, 0); - if (oldflags < 0) res = oldflags; - if (res == 0) - res = fcntl(write_fd, F_SETFD, oldflags | FD_CLOEXEC); - } - } - saved_errno = errno; - /* Close fds we tried for the write end that were too low. */ - for (fds_to_close_idx=0; fds_to_close_idx < 3; ++fds_to_close_idx) { - int temp_fd = fds_to_close[fds_to_close_idx]; - while (temp_fd >= 0 && close(temp_fd) < 0 && errno == EINTR); - } - errno = saved_errno; /* report dup or fcntl errors, not close. */ - fds[1] = write_fd; - } /* end if write fd was too small */ - - if (res != 0) - return PyErr_SetFromErrno(PyExc_OSError); - return Py_BuildValue("(ii)", fds[0], fds[1]); -} - /* module level code ********************************************************/ PyDoc_STRVAR(module_doc, @@ -811,7 +770,6 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"fork_exec", subprocess_fork_exec, METH_VARARGS, subprocess_fork_exec_doc}, - {"cloexec_pipe", subprocess_cloexec_pipe, METH_NOARGS, subprocess_cloexec_pipe_doc}, {NULL, NULL} /* sentinel */ }; @@ -827,11 +785,5 @@ static struct PyModuleDef _posixsubprocessmodule = { PyMODINIT_FUNC PyInit__posixsubprocess(void) { -#ifdef _SC_OPEN_MAX - max_fd = sysconf(_SC_OPEN_MAX); - if (max_fd == -1) -#endif - max_fd = 256; /* Matches Lib/subprocess.py */ - return PyModule_Create(&_posixsubprocessmodule); } |