diff options
| -rw-r--r-- | Doc/library/subprocess.rst | 41 | ||||
| -rw-r--r-- | Include/abstract.h | 3 | ||||
| -rw-r--r-- | Include/longobject.h | 17 | ||||
| -rw-r--r-- | Include/pythonrun.h | 3 | ||||
| -rw-r--r-- | Lib/subprocess.py | 305 | ||||
| -rw-r--r-- | Lib/test/test_subprocess.py | 85 | ||||
| -rw-r--r-- | Modules/_posixsubprocess.c | 385 | ||||
| -rw-r--r-- | Modules/posixmodule.c | 17 | ||||
| -rw-r--r-- | Objects/abstract.c | 60 | ||||
| -rw-r--r-- | Python/pythonrun.c | 26 | ||||
| -rw-r--r-- | setup.py | 3 |
11 files changed, 816 insertions, 129 deletions
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index a7df536..aeff9f0 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -28,7 +28,7 @@ Using the subprocess Module This module defines one class called :class:`Popen`: -.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0) +.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False) Arguments are: @@ -41,7 +41,8 @@ This module defines one class called :class:`Popen`: name for the executing program in utilities such as :program:`ps`. On Unix, with *shell=False* (default): In this case, the Popen class uses - :meth:`os.execvp` to execute the child program. *args* should normally be a + :meth:`os.execvp` like behavior to execute the child program. + *args* should normally be a sequence. If a string is specified for *args*, it will be used as the name or path of the program to execute; this will only work if the program is being given no arguments. @@ -108,7 +109,23 @@ This module defines one class called :class:`Popen`: applications should be captured into the same file handle as for stdout. If *preexec_fn* is set to a callable object, this object will be called in the - child process just before the child is executed. (Unix only) + child process just before the child is executed. + (Unix only) + + .. warning:: + + The *preexec_fn* parameter is not safe to use in the presence of threads + in your application. The child process could deadlock before exec is + called. + If you must use it, keep it trivial! Minimize the number of libraries + you call into. + + .. note:: + + If you need to modify the environment for the child use the *env* + parameter rather than doing it in a *preexec_fn*. + The *start_new_session* parameter can take the place of a previously + common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and :const:`2` will be closed before the child process is executed. (Unix only). @@ -124,9 +141,23 @@ This module defines one class called :class:`Popen`: searching the executable, so you can't specify the program's path relative to *cwd*. + If *restore_signals* is True (the default) all signals that Python has set to + SIG_IGN are restored to SIG_DFL in the child process before the exec. + Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. + (Unix only) + + .. versionchanged:: 3.2 + *restore_signals* was added. + + If *start_new_session* is True the setsid() system call will be made in the + child process prior to the execution of the subprocess. (Unix only) + + .. versionchanged:: 3.2 + *start_new_session* was added. + If *env* is not ``None``, it must be a mapping that defines the environment - variables for the new process; these are used instead of inheriting the current - process' environment, which is the default behavior. + variables for the new process; these are used instead of the default + behavior of inheriting the current process' environment. .. note:: diff --git a/Include/abstract.h b/Include/abstract.h index 6d40ef7..d2e9682 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -1232,6 +1232,9 @@ PyAPI_FUNC(int) _PyObject_RealIsInstance(PyObject *inst, PyObject *cls); PyAPI_FUNC(int) _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); +PyAPI_FUNC(char *const *) _PySequence_BytesToCharpArray(PyObject* self); + +PyAPI_FUNC(void) _Py_FreeCharPArray(char *const array[]); #ifdef __cplusplus } diff --git a/Include/longobject.h b/Include/longobject.h index 83d780d..4dc2e18 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -41,6 +41,23 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); #define PyLong_AsSocket_t(fd) (SOCKET_T)PyLong_AsLongLong(fd) #endif +/* Issue #1983: pid_t can be longer than a C long on some systems */ +#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT +#define PARSE_PID "i" +#define PyLong_FromPid PyLong_FromLong +#define PyLong_AsPid PyLong_AsLong +#elif SIZEOF_PID_T == SIZEOF_LONG +#define PARSE_PID "l" +#define PyLong_FromPid PyLong_FromLong +#define PyLong_AsPid PyLong_AsLong +#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG +#define PARSE_PID "L" +#define PyLong_FromPid PyLong_FromLongLong +#define PyLong_AsPid PyLong_AsLongLong +#else +#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)" +#endif /* SIZEOF_PID_T */ + /* For use by intobject.c only */ PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 09d3ee5..b49ad96 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -81,6 +81,9 @@ PyAPI_FUNC(int) Py_AtExit(void (*func)(void)); PyAPI_FUNC(void) Py_Exit(int); +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ +PyAPI_FUNC(void) _Py_RestoreSignals(void); + PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); /* Bootstrap */ diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 63ca956..b33601a 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -29,7 +29,8 @@ class Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False): Arguments are: @@ -72,8 +73,11 @@ parent. Additionally, stderr can be STDOUT, which indicates that the stderr data from the applications should be captured into the same file handle as for stdout. -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. +On UNIX, if preexec_fn is set to a callable object, this object will be +called in the child process just before the child is executed. The use +of preexec_fn is not thread safe, using it in the presence of threads +could lead to a deadlock in the child process before the new executable +is executed. If close_fds is true, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. @@ -84,6 +88,14 @@ shell. If cwd is not None, the current directory will be changed to cwd before the child is executed. +On UNIX, if restore_signals is True all signals that Python sets to +SIG_IGN are restored to SIG_DFL in the child process before the exec. +Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This +parameter does nothing on Windows. + +On UNIX, if start_new_session is True, the setsid() system call will be made +in the child process prior to executing the command. + If env is not None, it defines the environment variables for the new process. @@ -326,6 +338,7 @@ import os import traceback import gc import signal +import builtins # Exception classes used by this module. class CalledProcessError(Exception): @@ -375,6 +388,15 @@ else: import fcntl import pickle + try: + import _posixsubprocess + except ImportError: + _posixsubprocess = None + import warnings + warnings.warn("The _posixsubprocess module is not being used. " + "Child process reliability may suffer if your " + "program uses threads.", RuntimeWarning) + # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. @@ -596,7 +618,8 @@ class Popen(object): stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False): """Create new Popen instance.""" _cleanup() @@ -642,7 +665,7 @@ class Popen(object): # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None + # are -1 when not using PIPEs. The child objects are -1 # when not redirecting. (p2cread, p2cwrite, @@ -654,7 +677,8 @@ class Popen(object): startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) + errread, errwrite, + restore_signals, start_new_session) if mswindows: if p2cwrite is not None: @@ -666,15 +690,15 @@ class Popen(object): if bufsize == 0: bufsize = 1 # Nearly unbuffered (XXX for now) - if p2cwrite is not None: + if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) if self.universal_newlines: self.stdin = io.TextIOWrapper(self.stdin) - if c2pread is not None: + if c2pread != -1: self.stdout = io.open(c2pread, 'rb', bufsize) if universal_newlines: self.stdout = io.TextIOWrapper(self.stdout) - if errread is not None: + if errread != -1: self.stderr = io.open(errread, 'rb', bufsize) if universal_newlines: self.stderr = io.TextIOWrapper(self.stderr) @@ -739,11 +763,11 @@ class Popen(object): p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (-1, -1, -1, -1, -1, -1) - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) @@ -819,7 +843,8 @@ class Popen(object): startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" if not isinstance(args, str): @@ -973,9 +998,9 @@ class Popen(object): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: pass @@ -1034,7 +1059,8 @@ class Popen(object): startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + restore_signals, start_new_session): """Execute program (POSIX version)""" if isinstance(args, str): @@ -1048,113 +1074,190 @@ class Popen(object): if executable is None: executable = args[0] - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. + # For transferring possible exec failure from child to parent. + # Data format: "exception name:hex errno:description" + # Pickle is not used; it is complex and involves memory allocation. errpipe_read, errpipe_write = os.pipe() try: try: self._set_cloexec_flag(errpipe_write) - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child + if _posixsubprocess: + fs_encoding = sys.getfilesystemencoding() + def fs_encode(s): + """Encode s for use in the env, fs or cmdline.""" + return s.encode(fs_encoding, 'surrogateescape') + + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + + if env: + env_list = [fs_encode(k) + b'=' + fs_encode(v) + for k, v in env.items()] + else: + env_list = None # Use execv instead of execve. + if os.path.dirname(executable): + executable_list = (fs_encode(executable),) + else: + # This matches the behavior of os._execvpe(). + path_list = os.get_exec_path(env) + executable_list = (os.path.join(dir, executable) + for dir in path_list) + executable_list = tuple(fs_encode(exe) + for exe in executable_list) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + else: + # Pure Python implementation: It is not thread safe. + # This implementation may deadlock in the child if your + # parent process has any other threads running. + + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. See issue1336 + gc.disable() try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - if p2cread is not None: - os.dup2(p2cread, 0) - if c2pwrite is not None: - os.dup2(c2pwrite, 1) - if errwrite is not None: - os.dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - if p2cread is not None and p2cread not in (0,): - os.close(p2cread) - if c2pwrite is not None and \ - c2pwrite not in (p2cread, 1): - os.close(c2pwrite) - if (errwrite is not None and - errwrite not in (p2cread, c2pwrite, 2)): - os.close(errwrite) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - + self.pid = os.fork() except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so - # it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + if gc_was_enabled: + gc.enable() + raise + self._child_created = True + if self.pid == 0: + # Child + try: + # Close parent's pipe ends + if p2cwrite != -1: + os.close(p2cwrite) + if c2pread != -1: + os.close(c2pread) + if errread != -1: + os.close(errread) + os.close(errpipe_read) + + # Dup fds for child + if p2cread != -1: + os.dup2(p2cread, 0) + if c2pwrite != -1: + os.dup2(c2pwrite, 1) + if errwrite != -1: + os.dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + if p2cread != -1 and p2cread not in (0,): + os.close(p2cread) + if (c2pwrite != -1 and + c2pwrite not in (p2cread, 1)): + os.close(c2pwrite) + if (errwrite != -1 and + errwrite not in (p2cread, c2pwrite, 2)): + os.close(errwrite) + + # Close all other fds, if asked for + if close_fds: + self._close_fds(but=errpipe_write) + + if cwd is not None: + os.chdir(cwd) + + # This is a copy of Python/pythonrun.c + # _Py_RestoreSignals(). If that were exposed + # as a sys._py_restoresignals func it would be + # better.. but this pure python implementation + # isn't likely to be used much anymore. + if restore_signals: + signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') + for sig in signals: + if hasattr(signal, sig): + signal.signal(getattr(signal, sig), + signal.SIG_DFL) + + if start_new_session and hasattr(os, 'setsid'): + os.setsid() + + if preexec_fn: + preexec_fn() + + if env is None: + os.execvp(executable, args) + else: + os.execvpe(executable, args, env) + + except: + try: + exc_type, exc_value = sys.exc_info()[:2] + if isinstance(exc_value, OSError): + errno = exc_value.errno + else: + errno = 0 + message = '%s:%x:%s' % (exc_type.__name__, + errno, exc_value) + os.write(errpipe_write, message.encode()) + except: + # We MUST not allow anything odd happening + # above to prevent us from exiting below. + pass + + # This exitcode won't be reported to applications + # so it really doesn't matter what we return. + os._exit(255) + + # Parent + if gc_was_enabled: + gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: + if p2cread != -1 and p2cwrite != -1: os.close(p2cread) - if c2pwrite is not None and c2pread is not None: + if c2pwrite != -1 and c2pread != -1: os.close(c2pwrite) - if errwrite is not None and errread is not None: + if errwrite != -1 and errread != -1: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising an - # exception (limited to 1 MB) - data = _eintr_retry_call(os.read, errpipe_read, 1048576) + # exception (limited in size) + data = bytearray() + while True: + part = _eintr_retry_call(os.read, errpipe_read, 50000) + data += part + if not part or len(data) > 50000: + break finally: # be sure the FD is closed no matter what os.close(errpipe_read) if data: _eintr_retry_call(os.waitpid, self.pid, 0) - child_exception = pickle.loads(data) + try: + exception_name, hex_errno, err_msg = data.split(b':', 2) + except ValueError: + print('Bad exception data:', repr(data)) + exception_name = b'RuntimeError' + hex_errno = b'0' + err_msg = b'Unknown' + child_exception_type = getattr( + builtins, exception_name.decode('ascii'), + RuntimeError) for fd in (p2cwrite, c2pread, errread): - if fd is not None: + if fd != -1: os.close(fd) - raise child_exception + err_msg = err_msg.decode() + if issubclass(child_exception_type, OSError) and hex_errno: + errno = int(hex_errno, 16) + if errno != 0: + err_msg = os.strerror(errno) + raise child_exception_type(errno, err_msg) + raise child_exception_type(err_msg) def _handle_exitstatus(self, sts): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index dff9012..74d9ce4 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -568,12 +568,53 @@ class POSIXProcessTestCase(unittest.TestCase): self.assertFalse(subprocess._active, "subprocess._active not empty") def test_exceptions(self): - # caught & re-raised exceptions - with self.assertRaises(OSError) as c: + nonexistent_dir = "/_this/pa.th/does/not/exist" + try: + os.chdir(nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + else: + self.fail("chdir to nonexistant directory %s succeeded." % + nonexistent_dir) + + # Error in the child re-raised in the parent. + try: p = subprocess.Popen([sys.executable, "-c", ""], - cwd="/this/path/does/not/exist") - # The attribute child_traceback should contain "os.chdir" somewhere. - self.assertIn("os.chdir", c.exception.child_traceback) + cwd=nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_restore_signals(self): + # Code coverage for both values of restore_signals to make sure it + # at least does not blow up. + # A test for behavior would be complex. Contributions welcome. + subprocess.call([sys.executable, "-c", ""], restore_signals=True) + subprocess.call([sys.executable, "-c", ""], restore_signals=False) + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) def test_run_abort(self): # returncode handles signal termination @@ -584,7 +625,8 @@ class POSIXProcessTestCase(unittest.TestCase): self.assertEqual(-p.returncode, signal.SIGABRT) def test_preexec(self): - # preexec function + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' 'sys.stdout.write(os.getenv("FRUIT"))'], @@ -592,6 +634,22 @@ class POSIXProcessTestCase(unittest.TestCase): preexec_fn=lambda: os.putenv("FRUIT", "apple")) self.assertEqual(p.stdout.read(), b"apple") + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except RuntimeError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + def test_args_string(self): # args is a string fd, fname = mkstemp() @@ -836,6 +894,20 @@ class ProcessTestCaseNoPoll(ProcessTestCase): ProcessTestCase.tearDown(self) +@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), + "_posixsubprocess extension module not found.") +class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): + def setUp(self): + subprocess._posixsubprocess = None + ProcessTestCase.setUp(self) + POSIXProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._posixsubprocess = sys.modules['_posixsubprocess'] + POSIXProcessTestCase.tearDown(self) + ProcessTestCase.tearDown(self) + + class HelperFunctionTests(unittest.TestCase): @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") def test_eintr_retry_call(self): @@ -859,6 +931,7 @@ def test_main(): unit_tests = (ProcessTestCase, POSIXProcessTestCase, Win32ProcessTestCase, + ProcessTestCasePOSIXPurePython, CommandTests, ProcessTestCaseNoPoll, HelperFunctionTests) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c new file mode 100644 index 0000000..d388d89 --- /dev/null +++ b/Modules/_posixsubprocess.c @@ -0,0 +1,385 @@ +/* Authors: Gregory P. Smith & Jeffrey Yasskin */ +#include "Python.h" +#include <unistd.h> + + +#define POSIX_CALL(call) if ((call) == -1) goto error + + +/* Maximum file descriptor, initialized on module load. */ +static long max_fd; + + +/* Given the gc module call gc.enable() and return 0 on success. */ +static int _enable_gc(PyObject *gc_module) +{ + PyObject *result; + result = PyObject_CallMethod(gc_module, "enable", NULL); + if (result == NULL) + return 1; + Py_DECREF(result); + return 0; +} + + +/* + * This function is code executed in the child process immediately after fork + * to set things up and call exec(). + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + * + * This restriction is documented at + * http://www.opengroup.org/onlinepubs/009695399/functions/fork.html. + */ +static void child_exec(char *const exec_array[], + char *const argv[], + char *const envp[], + const char *cwd, + int p2cread, int p2cwrite, + int c2pread, int c2pwrite, + int errread, int errwrite, + int errpipe_read, int errpipe_write, + int close_fds, int restore_signals, + int call_setsid, + PyObject *preexec_fn, + PyObject *preexec_fn_args_tuple) +{ + int i, saved_errno, fd_num; + PyObject *result; + const char* err_msg; + /* Buffer large enough to hold a hex integer. We can't malloc. */ + char hex_errno[sizeof(saved_errno)*2+1]; + + /* Close parent's pipe ends. */ + if (p2cwrite != -1) { + POSIX_CALL(close(p2cwrite)); + } + if (c2pread != -1) { + POSIX_CALL(close(c2pread)); + } + if (errread != -1) { + POSIX_CALL(close(errread)); + } + POSIX_CALL(close(errpipe_read)); + + /* Dup fds for child. */ + if (p2cread != -1) { + POSIX_CALL(dup2(p2cread, 0)); /* stdin */ + } + if (c2pwrite != -1) { + POSIX_CALL(dup2(c2pwrite, 1)); /* stdout */ + } + if (errwrite != -1) { + POSIX_CALL(dup2(errwrite, 2)); /* stderr */ + } + + /* Close pipe fds. Make sure we don't close the same fd more than */ + /* once, or standard fds. */ + if (p2cread != -1 && p2cread != 0) { + POSIX_CALL(close(p2cread)); + } + if (c2pwrite != -1 && c2pwrite != p2cread && c2pwrite != 1) { + POSIX_CALL(close(c2pwrite)); + } + if (errwrite != -1 && errwrite != p2cread && + errwrite != c2pwrite && errwrite != 2) { + POSIX_CALL(close(errwrite)); + } + + /* close() is intentionally not checked for errors here as we are closing */ + /* a large range of fds, some of which may be invalid. */ + if (close_fds) { + for (fd_num = 3; fd_num < errpipe_write; ++fd_num) { + close(fd_num); + } + for (fd_num = errpipe_write+1; fd_num < max_fd; ++fd_num) { + close(fd_num); + } + } + + if (cwd) + POSIX_CALL(chdir(cwd)); + + if (restore_signals) + _Py_RestoreSignals(); + +#ifdef HAVE_SETSID + if (call_setsid) + POSIX_CALL(setsid()); +#endif + + if (preexec_fn != Py_None && preexec_fn_args_tuple) { + /* This is where the user has asked us to deadlock their program. */ + result = PyObject_Call(preexec_fn, preexec_fn_args_tuple, NULL); + if (result == NULL) { + /* Stringifying the exception or traceback would involve + * memory allocation and thus potential for deadlock. + * We've already faced potential deadlock by calling back + * into Python in the first place, so it probably doesn't + * matter but we avoid it to minimize the possibility. */ + err_msg = "Exception occurred in preexec_fn."; + errno = 0; /* We don't want to report an OSError. */ + goto error; + } + /* Py_DECREF(result); - We're about to exec so why bother? */ + } + + /* This loop matches the Lib/os.py _execvpe()'s PATH search when */ + /* given the executable_list generated by Lib/subprocess.py. */ + saved_errno = 0; + for (i = 0; exec_array[i] != NULL; ++i) { + const char *executable = exec_array[i]; + if (envp) { + execve(executable, argv, envp); + } else { + execv(executable, argv); + } + if (errno != ENOENT && errno != ENOTDIR && saved_errno == 0) { + saved_errno = errno; + } + } + /* Report the first exec error, not the last. */ + if (saved_errno) + errno = saved_errno; + +error: + saved_errno = errno; + /* Report the posix error to our parent process. */ + if (saved_errno) { + char *cur; + write(errpipe_write, "OSError:", 8); + cur = hex_errno + sizeof(hex_errno); + while (saved_errno != 0 && cur > hex_errno) { + *--cur = "0123456789ABCDEF"[saved_errno % 16]; + saved_errno /= 16; + } + write(errpipe_write, cur, hex_errno + sizeof(hex_errno) - cur); + write(errpipe_write, ":", 1); + /* We can't call strerror(saved_errno). It is not async signal safe. + * The parent process will look the error message up. */ + } else { + write(errpipe_write, "RuntimeError:0:", 15); + write(errpipe_write, err_msg, strlen(err_msg)); + } +} + + +static PyObject * +subprocess_fork_exec(PyObject* self, PyObject *args) +{ + PyObject *gc_module = NULL; + PyObject *executable_list, *py_close_fds; + PyObject *env_list, *preexec_fn; + PyObject *process_args = NULL, *converted_args = NULL; + PyObject *preexec_fn_args_tuple = NULL; + int p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite; + int errpipe_read, errpipe_write, close_fds, restore_signals; + int call_setsid; + const char *cwd; + pid_t pid; + int need_to_reenable_gc = 0; + char *const *exec_array, *const *argv = NULL, *const *envp = NULL; + Py_ssize_t arg_num; + + if (!PyArg_ParseTuple( + args, "OOOzOiiiiiiiiiiO:fork_exec", + &process_args, &executable_list, &py_close_fds, &cwd, &env_list, + &p2cread, &p2cwrite, &c2pread, &c2pwrite, + &errread, &errwrite, &errpipe_read, &errpipe_write, + &restore_signals, &call_setsid, &preexec_fn)) + return NULL; + + close_fds = PyObject_IsTrue(py_close_fds); + if (close_fds && errpipe_write < 3) { /* precondition */ + PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); + return NULL; + } + + /* We need to call gc.disable() when we'll be calling preexec_fn */ + if (preexec_fn != Py_None) { + PyObject *result; + gc_module = PyImport_ImportModule("gc"); + if (gc_module == NULL) + return NULL; + result = PyObject_CallMethod(gc_module, "isenabled", NULL); + if (result == NULL) + return NULL; + need_to_reenable_gc = PyObject_IsTrue(result); + Py_DECREF(result); + if (need_to_reenable_gc == -1) + return NULL; + result = PyObject_CallMethod(gc_module, "disable", NULL); + if (result == NULL) + return NULL; + Py_DECREF(result); + } + + exec_array = _PySequence_BytesToCharpArray(executable_list); + if (!exec_array) + return NULL; + + /* Convert args and env into appropriate arguments for exec() */ + /* These conversions are done in the parent process to avoid allocating + or freeing memory in the child process. */ + if (process_args != Py_None) { + /* Equivalent to: */ + /* tuple(PyUnicode_FSConverter(arg) for arg in process_args) */ + process_args = PySequence_Fast(process_args, "argv must be a tuple"); + converted_args = PyTuple_New(PySequence_Size(process_args)); + if (converted_args == NULL) + goto cleanup; + for (arg_num = 0; arg_num < PySequence_Size(process_args); ++arg_num) { + PyObject *borrowed_arg, *converted_arg; + borrowed_arg = PySequence_Fast_GET_ITEM(process_args, arg_num); + if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) + goto cleanup; + PyTuple_SET_ITEM(converted_args, arg_num, converted_arg); + } + + argv = _PySequence_BytesToCharpArray(converted_args); + Py_CLEAR(converted_args); + Py_CLEAR(process_args); + if (!argv) + goto cleanup; + } + + if (env_list != Py_None) { + envp = _PySequence_BytesToCharpArray(env_list); + if (!envp) + goto cleanup; + } + + if (preexec_fn != Py_None) { + preexec_fn_args_tuple = PyTuple_New(0); + if (!preexec_fn_args_tuple) + goto cleanup; + _PyImport_AcquireLock(); + } + + pid = fork(); + if (pid == 0) { + /* Child process */ + /* + * Code from here to _exit() must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ + + if (preexec_fn != Py_None) { + /* We'll be calling back into Python later so we need to do this. + * This call may not be async-signal-safe but neither is calling + * back into Python. The user asked us to use hope as a strategy + * to avoid deadlock... */ + PyOS_AfterFork(); + } + + child_exec(exec_array, argv, envp, cwd, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, errpipe_read, errpipe_write, + close_fds, restore_signals, call_setsid, + preexec_fn, preexec_fn_args_tuple); + _exit(255); + return NULL; /* Dead code to avoid a potential compiler warning. */ + } + if (pid == -1) { + /* Capture the errno exception before errno can be clobbered. */ + PyErr_SetFromErrno(PyExc_OSError); + } + if (preexec_fn != Py_None && + _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, + "not holding the import lock"); + } + + /* Parent process */ + if (envp) + _Py_FreeCharPArray(envp); + if (argv) + _Py_FreeCharPArray(argv); + _Py_FreeCharPArray(exec_array); + + /* Reenable gc in the parent process (or if fork failed). */ + if (need_to_reenable_gc && _enable_gc(gc_module)) { + Py_XDECREF(gc_module); + return NULL; + } + Py_XDECREF(gc_module); + + if (pid == -1) + return NULL; /* fork() failed. Exception set earlier. */ + + return PyLong_FromPid(pid); + +cleanup: + if (envp) + _Py_FreeCharPArray(envp); + if (argv) + _Py_FreeCharPArray(argv); + _Py_FreeCharPArray(exec_array); + Py_XDECREF(converted_args); + Py_XDECREF(process_args); + + /* Reenable gc if it was disabled. */ + if (need_to_reenable_gc) + _enable_gc(gc_module); + Py_XDECREF(gc_module); + return NULL; +} + + +PyDoc_STRVAR(subprocess_fork_exec_doc, +"fork_exec(args, executable_list, close_fds, cwd, env,\n\ + p2cread, p2cwrite, c2pread, c2pwrite,\n\ + errread, errwrite, errpipe_read, errpipe_write,\n\ + restore_signals, call_setsid, preexec_fn)\n\ +\n\ +Forks a child process, closes parent file descriptors as appropriate in the\n\ +child and dups the few that are needed before calling exec() in the child\n\ +process.\n\ +\n\ +The preexec_fn, if supplied, will be called immediately before exec.\n\ +WARNING: preexec_fn is NOT SAFE if your application uses threads.\n\ + It may trigger infrequent, difficult to debug deadlocks.\n\ +\n\ +If an error occurs in the child process before the exec, it is\n\ +serialized and written to the errpipe_write fd per subprocess.py.\n\ +\n\ +Returns: the child process's PID.\n\ +\n\ +Raises: Only on an error in the parent process.\n\ +"); + + +/* module level code ********************************************************/ + +PyDoc_STRVAR(module_doc, +"A POSIX helper for the subprocess module."); + + +static PyMethodDef module_methods[] = { + {"fork_exec", subprocess_fork_exec, METH_VARARGS, subprocess_fork_exec_doc}, + {NULL, NULL} /* sentinel */ +}; + + +static struct PyModuleDef _posixsubprocessmodule = { + PyModuleDef_HEAD_INIT, + "_posixsubprocess", + module_doc, + -1, /* No memory is needed. */ + module_methods, +}; + +PyMODINIT_FUNC +PyInit__posixsubprocess(void) +{ +#ifdef _SC_OPEN_MAX + max_fd = sysconf(_SC_OPEN_MAX); + if (max_fd == -1) +#endif + max_fd = 256; /* Matches Lib/subprocess.py */ + + return PyModule_Create(&_posixsubprocessmodule); +} diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index abe6189..a18ef8e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -303,23 +303,6 @@ extern int lstat(const char *, struct stat *); #define WAIT_STATUS_INT(s) (s) #endif /* UNION_WAIT */ -/* Issue #1983: pid_t can be longer than a C long on some systems */ -#if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT -#define PARSE_PID "i" -#define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong -#elif SIZEOF_PID_T == SIZEOF_LONG -#define PARSE_PID "l" -#define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong -#elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG -#define PARSE_PID "L" -#define PyLong_FromPid PyLong_FromLongLong -#define PyLong_AsPid PyLong_AsLongLong -#else -#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)" -#endif /* SIZEOF_PID_T */ - /* Don't use the "_r" form if we don't need it (also, won't have a prototype for it, at least on Solaris -- maybe others as well?). */ #if defined(HAVE_CTERMID_R) && defined(WITH_THREAD) diff --git a/Objects/abstract.c b/Objects/abstract.c index d4cba74..952ad40 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2722,3 +2722,63 @@ PyIter_Next(PyObject *iter) PyErr_Clear(); return result; } + + +/* + * Flatten a sequence of bytes() objects into a C array of + * NULL terminated string pointers with a NULL char* terminating the array. + * (ie: an argv or env list) + * + * Memory allocated for the returned list is allocated using malloc() and MUST + * be freed by the caller using a free() loop or _Py_FreeCharPArray(). + */ +char *const * +_PySequence_BytesToCharpArray(PyObject* self) +{ + char **array; + Py_ssize_t i, argc; + + argc = PySequence_Size(self); + if (argc == -1) + return NULL; + + array = malloc((argc + 1) * sizeof(char *)); + if (array == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (i = 0; i < argc; ++i) { + char *data; + PyObject *item = PySequence_GetItem(self, i); + data = PyBytes_AsString(item); + if (data == NULL) { + /* NULL terminate before freeing. */ + array[i] = NULL; + goto fail; + } + array[i] = strdup(data); + if (!array[i]) { + PyErr_NoMemory(); + goto fail; + } + } + array[argc] = NULL; + + return array; + +fail: + _Py_FreeCharPArray(array); + return NULL; +} + + +/* Free's a NULL terminated char** array of C strings. */ +void +_Py_FreeCharPArray(char *const array[]) +{ + Py_ssize_t i; + for (i = 0; array[i] != NULL; ++i) { + free(array[i]); + } + free((void*)array); +} diff --git a/Python/pythonrun.c b/Python/pythonrun.c index cfa19b0..2bdef98 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -2130,6 +2130,27 @@ initsigs(void) } +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} + + /* * The file descriptor fd is considered ``interactive'' if either * a) isatty(fd) is TRUE, or @@ -2223,6 +2244,11 @@ PyOS_getsig(int sig) #endif } +/* + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) { @@ -546,6 +546,9 @@ class PyBuildExt(build_ext): # CSV files exts.append( Extension('_csv', ['_csv.c']) ) + # POSIX subprocess module helper. + exts.append( Extension('_posixsubprocess', ['_posixsubprocess.c']) ) + # socket(2) exts.append( Extension('_socket', ['socketmodule.c'], depends = ['socketmodule.h']) ) |