diff options
| -rw-r--r-- | Doc/library/subprocess.rst | 289 | ||||
| -rw-r--r-- | Doc/whatsnew/3.5.rst | 8 | ||||
| -rw-r--r-- | Lib/subprocess.py | 124 | ||||
| -rw-r--r-- | Lib/test/test_subprocess.py | 97 | ||||
| -rw-r--r-- | Misc/NEWS | 3 |
5 files changed, 386 insertions, 135 deletions
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 49a82bd..4fef6ea 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -25,160 +25,99 @@ modules and functions can be found in the following sections. Using the :mod:`subprocess` Module ---------------------------------- -The recommended approach to invoking subprocesses is to use the following -convenience functions for all use cases they can handle. For more advanced +The recommended approach to invoking subprocesses is to use the :func:`run` +function for all use cases it can handle. For more advanced use cases, the underlying :class:`Popen` interface can be used directly. +The :func:`run` function was added in Python 3.5; if you need to retain +compatibility with older versions, see the :ref:`call-function-trio` section. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + +.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ + shell=False, timeout=None, check=False) Run the command described by *args*. Wait for command to complete, then - return the :attr:`returncode` attribute. + return a :class:`CompletedProcess` instance. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the use of keyword-only notation in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. + same as that of the :class:`Popen` constructor - apart from *timeout*, + *input* and *check*, all the arguments to this function are passed through to + that interface. - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout - expires, the child process will be killed and then waited for again. The + This does not capture stdout or stderr by default. To do so, pass + :data:`PIPE` for the *stdout* and/or *stderr* arguments. + + The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout + expires, the child process will be killed and waited for. The :exc:`TimeoutExpired` exception will be re-raised after the child process has terminated. - Examples:: - - >>> subprocess.call(["ls", "-l"]) - 0 - - >>> subprocess.call("exit 1", shell=True) - 1 - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) - - Run command with arguments. Wait for command to complete. If the return - code was zero then return, otherwise raise :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute. - - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. + The *input* argument is passed to :meth:`Popen.communicate` and thus to the + subprocess's stdin. If used it must be a byte sequence, or a string if + ``universal_newlines=True``. When used, the internal :class:`Popen` object + is automatically created with ``stdin=PIPE``, and the *stdin* argument may + not be used as well. - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout - expires, the child process will be killed and then waited for again. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. + If *check* is True, and the process exits with a non-zero exit code, a + :exc:`CalledProcessError` exception will be raised. Attributes of that + exception hold the arguments, the exit code, and stdout and stderr if they + were captured. Examples:: - >>> subprocess.check_call(["ls", "-l"]) - 0 + >>> subprocess.run(["ls", "-l"]) # doesn't capture output + CompletedProcess(args=['ls', '-l'], returncode=0) - >>> subprocess.check_call("exit 1", shell=True) + >>> subprocess.run("exit 1", shell=True, check=True) Traceback (most recent call last): - ... + ... subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - .. note:: + >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) + CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, + stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. + .. versionadded:: 3.5 - .. versionchanged:: 3.3 - *timeout* was added. +.. class:: CompletedProcess + The return value from :func:`run`, representing a process that has finished. -.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) + .. attribute:: args - Run command with arguments and return its output. + The arguments used to launch the process. This may be a list or a string. - If the return code was non-zero it raises a :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute and any output in the - :attr:`~CalledProcessError.output` attribute. + .. attribute:: returncode - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - this functions passes all - supplied arguments other than *input* and *timeout* directly through to - that interface. In addition, *stdout* is not permitted as an argument, as - it is used internally to collect the output from the subprocess. + Exit status of the child process. Typically, an exit status of 0 indicates + that it ran successfully. - The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout - expires, the child process will be killed and then waited for again. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. + A negative value ``-N`` indicates that the child was terminated by signal + ``N`` (POSIX only). - The *input* argument is passed to :meth:`Popen.communicate` and thus to the - subprocess's stdin. If used it must be a byte sequence, or a string if - ``universal_newlines=True``. When used, the internal :class:`Popen` object - is automatically created with ``stdin=PIPE``, and the *stdin* argument may - not be used as well. + .. attribute:: stdout - Examples:: + Captured stdout from the child process. A bytes sequence, or a string if + :func:`run` was called with ``universal_newlines=True``. None if stdout + was not captured. - >>> subprocess.check_output(["echo", "Hello World!"]) - b'Hello World!\n' + If you ran the process with ``stderr=subprocess.STDOUT``, stdout and + stderr will be combined in this attribute, and :attr:`stderr` will be + None. - >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True) - 'Hello World!\n' + .. attribute:: stderr - >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"], - ... input=b"when in the course of fooman events\n") - b'when in the course of barman events\n' + Captured stderr from the child process. A bytes sequence, or a string if + :func:`run` was called with ``universal_newlines=True``. None if stderr + was not captured. - >>> subprocess.check_output("exit 1", shell=True) - Traceback (most recent call last): - ... - subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 + .. method:: check_returncode() - By default, this function will return the data as encoded bytes. The actual - encoding of the output data may depend on the command being invoked, so the - decoding to text will often need to be handled at the application level. + If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`. - This behaviour may be overridden by setting *universal_newlines* to - ``True`` as described below in :ref:`frequently-used-arguments`. - - To also capture standard error in the result, use - ``stderr=subprocess.STDOUT``:: - - >>> subprocess.check_output( - ... "ls non_existent_file; exit 0", - ... stderr=subprocess.STDOUT, - ... shell=True) - 'ls: non_existent_file: No such file or directory\n' - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.4 - *input* was added. + .. versionadded:: 3.5 .. data:: DEVNULL @@ -225,11 +164,22 @@ use cases, the underlying :class:`Popen` interface can be used directly. .. attribute:: output - Output of the child process if this exception is raised by + Output of the child process if it was captured by :func:`run` or :func:`check_output`. Otherwise, ``None``. + .. attribute:: stdout + + Alias for output, for symmetry with :attr:`stderr`. + + .. attribute:: stderr + + Stderr output of the child process if it was captured by :func:`run`. + Otherwise, ``None``. + .. versionadded:: 3.3 + .. versionchanged:: 3.5 + *stdout* and *stderr* attributes added .. exception:: CalledProcessError @@ -246,9 +196,20 @@ use cases, the underlying :class:`Popen` interface can be used directly. .. attribute:: output - Output of the child process if this exception is raised by + Output of the child process if it was captured by :func:`run` or :func:`check_output`. Otherwise, ``None``. + .. attribute:: stdout + + Alias for output, for symmetry with :attr:`stderr`. + + .. attribute:: stderr + + Stderr output of the child process if it was captured by :func:`run`. + Otherwise, ``None``. + + .. versionchanged:: 3.5 + *stdout* and *stderr* attributes added .. _frequently-used-arguments: @@ -852,6 +813,96 @@ The :mod:`subprocess` module exposes the following constants. This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified. +.. _call-function-trio: + +Older high-level API +-------------------- + +Prior to Python 3.5, these three functions comprised the high level API to +subprocess. You can now use :func:`run` in many cases, but lots of existing code +calls these functions. + +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + + Run the command described by *args*. Wait for command to complete, then + return the :attr:`returncode` attribute. + + This is equivalent to:: + + run(...).returncode + + (except that the *input* and *check* parameters are not supported) + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionchanged:: 3.3 + *timeout* was added. + +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + + Run command with arguments. Wait for command to complete. If the return + code was zero then return, otherwise raise :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`~CalledProcessError.returncode` attribute. + + This is equivalent to:: + + run(..., check=True) + + (except that the *input* parameter is not supported) + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this + function. The child process will block if it generates enough + output to a pipe to fill up the OS pipe buffer as the pipes are + not being read from. + + .. versionchanged:: 3.3 + *timeout* was added. + + +.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) + + Run command with arguments and return its output. + + If the return code was non-zero it raises a :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`~CalledProcessError.returncode` attribute and any output in the + :attr:`~CalledProcessError.output` attribute. + + This is equivalent to:: + + run(..., check=True, stdout=PIPE).stdout + + By default, this function will return the data as encoded bytes. The actual + encoding of the output data may depend on the command being invoked, so the + decoding to text will often need to be handled at the application level. + + This behaviour may be overridden by setting *universal_newlines* to + ``True`` as described above in :ref:`frequently-used-arguments`. + + To also capture standard error in the result, use + ``stderr=subprocess.STDOUT``:: + + >>> subprocess.check_output( + ... "ls non_existent_file; exit 0", + ... stderr=subprocess.STDOUT, + ... shell=True) + 'ls: non_existent_file: No such file or directory\n' + + .. versionadded:: 3.1 + + .. versionchanged:: 3.3 + *timeout* was added. + + .. versionchanged:: 3.4 + *input* was added. .. _subprocess-replacements: diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index a1130df..e865402 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -492,6 +492,14 @@ tarfile * The :func:`tarfile.open` function now supports ``'x'`` (exclusive creation) mode. (Contributed by Berker Peksag in :issue:`21717`.) +subprocess +---------- + +* The new :func:`subprocess.run` function runs subprocesses and returns a + :class:`subprocess.CompletedProcess` object. It Provides a more consistent + API than :func:`~subprocess.call`, :func:`~subprocess.check_call` and + :func:`~subprocess.check_output`. + time ---- diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e92928e..b6c4374 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -377,27 +377,51 @@ class CalledProcessError(SubprocessError): The exit status will be stored in the returncode attribute; check_output() will also store the output in the output attribute. """ - def __init__(self, returncode, cmd, output=None): + def __init__(self, returncode, cmd, output=None, stderr=None): self.returncode = returncode self.cmd = cmd self.output = output + self.stderr = stderr + def __str__(self): return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + @property + def stdout(self): + """Alias for output attribute, to match stderr""" + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value + class TimeoutExpired(SubprocessError): """This exception is raised when the timeout expires while waiting for a child process. """ - def __init__(self, cmd, timeout, output=None): + def __init__(self, cmd, timeout, output=None, stderr=None): self.cmd = cmd self.timeout = timeout self.output = output + self.stderr = stderr def __str__(self): return ("Command '%s' timed out after %s seconds" % (self.cmd, self.timeout)) + @property + def stdout(self): + return self.output + + @stdout.setter + def stdout(self, value): + # There's no obvious reason to set this, but allow it anyway so + # .stdout is a transparent alias for .output + self.output = value + if _mswindows: import threading @@ -433,8 +457,8 @@ else: __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", - "getoutput", "check_output", "CalledProcessError", "DEVNULL", - "SubprocessError", "TimeoutExpired"] + "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL", + "SubprocessError", "TimeoutExpired", "CompletedProcess"] # NOTE: We intentionally exclude list2cmdline as it is # considered an internal implementation detail. issue10838. @@ -595,29 +619,97 @@ def check_output(*popenargs, timeout=None, **kwargs): """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') - if 'input' in kwargs: + + if 'input' in kwargs and kwargs['input'] is None: + # Explicitly passing input=None was previously equivalent to passing an + # empty string. That is maintained here for backwards compatibility. + kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' + + return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, + **kwargs).stdout + + +class CompletedProcess(object): + """A process that has finished running. + + This is returned by run(). + + Attributes: + args: The list or str args passed to run(). + returncode: The exit code of the process, negative for signals. + stdout: The standard output (None if not captured). + stderr: The standard error (None if not captured). + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + def check_returncode(self): + """Raise CalledProcessError if the exit code is non-zero.""" + if self.returncode: + raise CalledProcessError(self.returncode, self.args, self.stdout, + self.stderr) + + +def run(*popenargs, input=None, timeout=None, check=False, **kwargs): + """Run command with arguments and return a CompletedProcess instance. + + The returned instance will have attributes args, returncode, stdout and + stderr. By default, stdout and stderr are not captured, and those attributes + will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. + + If check is True and the exit code was non-zero, it raises a + CalledProcessError. The CalledProcessError object will have the return code + in the returncode attribute, and output & stderr attributes if those streams + were captured. + + If timeout is given, and the process takes too long, a TimeoutExpired + exception will be raised. + + There is an optional argument "input", allowing you to + pass a string to the subprocess's stdin. If you use this argument + you may not also use the Popen constructor's "stdin" argument, as + it will be used internally. + + The other arguments are the same as for the Popen constructor. + + If universal_newlines=True is passed, the "input" argument must be a + string and stdout/stderr in the returned object will be strings rather than + bytes. + """ + if input is not None: if 'stdin' in kwargs: raise ValueError('stdin and input arguments may not both be used.') - inputdata = kwargs['input'] - del kwargs['input'] kwargs['stdin'] = PIPE - else: - inputdata = None - with Popen(*popenargs, stdout=PIPE, **kwargs) as process: + + with Popen(*popenargs, **kwargs) as process: try: - output, unused_err = process.communicate(inputdata, timeout=timeout) + stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired: process.kill() - output, unused_err = process.communicate() - raise TimeoutExpired(process.args, timeout, output=output) + stdout, stderr = process.communicate() + raise TimeoutExpired(process.args, timeout, output=stdout, + stderr=stderr) except: process.kill() process.wait() raise retcode = process.poll() - if retcode: - raise CalledProcessError(retcode, process.args, output=output) - return output + if check and retcode: + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) + return CompletedProcess(process.args, retcode, stdout, stderr) def list2cmdline(seq): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 7b66945..7398bdc 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1232,6 +1232,102 @@ class ProcessTestCase(BaseTestCase): fds_after_exception = os.listdir(fd_directory) self.assertEqual(fds_before_popen, fds_after_exception) + +class RunFuncTestCase(BaseTestCase): + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + output = self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = self.run_python(( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase): @@ -2542,6 +2638,7 @@ def test_main(): ProcessTestCaseNoPoll, CommandsWithSpaces, ContextManagerTests, + RunFuncTestCase, ) support.run_unittest(*unit_tests) @@ -32,6 +32,9 @@ Core and Builtins Library ------- +- Issue #23342: Add a subprocess.run() function than returns a CalledProcess + instance for a more consistent API than the existing call* functions. + - Issue #21217: inspect.getsourcelines() now tries to compute the start and end lines from the code object, fixing an issue when a lambda function is used as decorator argument. Patch by Thomas Ballinger. |