From a1ed539268e37b12a4a864738b4d1e12bdb92793 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 23 Mar 2013 11:44:25 -0700 Subject: Fixes issue #17488: Change the subprocess.Popen bufsize parameter default value from unbuffered (0) to buffering (-1) to match the behavior existing code expects and match the behavior of the subprocess module in Python 2 to avoid introducing hard to track down bugs. --- Doc/library/subprocess.rst | 23 +++++++++++++---------- Lib/subprocess.py | 18 +++++++++--------- Lib/test/test_subprocess.py | 22 ++++++++++++++++++++++ Misc/NEWS | 5 +++++ 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 1c8aafd..59ee13f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -284,7 +284,7 @@ are able to handle the less common cases not covered by the convenience functions. -.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, \ +.. class:: Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, \ stderr=None, preexec_fn=None, close_fds=True, shell=False, \ cwd=None, env=None, universal_newlines=False, \ startupinfo=None, creationflags=0, restore_signals=True, \ @@ -356,17 +356,20 @@ functions. untrusted input. See the warning under :ref:`frequently-used-arguments` for details. - *bufsize*, if given, has the same meaning as the corresponding argument to the - built-in open() function: :const:`0` means unbuffered, :const:`1` means line - buffered, any other positive value means use a buffer of (approximately) that - size. A negative *bufsize* means to use the system default, which usually means - fully buffered. The default value for *bufsize* is :const:`0` (unbuffered). + *bufsize* will be supplied as the corresponding argument to the :meth:`io.open` + function when creating the stdin/stdout/stderr pipe file objects: + :const:`0` means unbuffered (read and write are one system call and can return short), + :const:`1` means line buffered, any other positive value means use a buffer of + approximately that size. A negative bufsize (the default) means + the system default of io.DEFAULT_BUFFER_SIZE will be used. - .. note:: + .. versionchanged:: 3.2.4 - If you experience performance issues, it is recommended that you try to - enable buffering by setting *bufsize* to either -1 or a large enough - positive value (such as 4096). + *bufsize* now defaults to -1 to enable buffering by default to match the + behavior that most code expects. In 3.2.0 through 3.2.3 it incorrectly + defaulted to :const:`0` which was unbuffered and allowed short reads. + This was unintentional and did not match the behavior of Python 2 as + most code expected. The *executable* argument specifies a replacement program to execute. It is very seldom needed. When ``shell=False``, *executable* replaces the diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 7cfe5df..7255645 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -25,7 +25,7 @@ Using the subprocess module =========================== This module defines one class called Popen: -class Popen(args, bufsize=0, executable=None, +class Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, @@ -56,12 +56,12 @@ not all MS Windows applications interpret the command line the same way: The list2cmdline is designed for applications using the same rules as the MS C runtime. -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). +bufsize will be supplied as the corresponding argument to the io.open() +function when creating the stdin/stdout/stderr pipe file objects: +0 means unbuffered (read & write are one system call and can return short), +1 means line buffered, any other positive value means use a buffer of +approximately that size. A negative bufsize, the default, means the system +default of io.DEFAULT_BUFFER_SIZE will be used. stdin, stdout and stderr specify the executed programs' standard input, standard output and standard error file handles, respectively. @@ -638,7 +638,7 @@ _PLATFORM_DEFAULT_CLOSE_FDS = object() class Popen(object): - def __init__(self, args, bufsize=0, executable=None, + def __init__(self, args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, cwd=None, env=None, universal_newlines=False, @@ -650,7 +650,7 @@ class Popen(object): self._child_created = False if bufsize is None: - bufsize = 0 # Restore default + bufsize = -1 # Restore default if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index b1e9027..1a50de3 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -79,6 +79,28 @@ class PopenExecuteChildRaises(subprocess.Popen): class ProcessTestCase(BaseTestCase): + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.wait() + def test_call_seq(self): # call() function with sequence argument rc = subprocess.call([sys.executable, "-c", diff --git a/Misc/NEWS b/Misc/NEWS index 6a36c3a..e4d04b1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -233,6 +233,11 @@ Core and Builtins Library ------- +- Issue #17488: Change the subprocess.Popen bufsize parameter default value + from unbuffered (0) to buffering (-1) to match the behavior existing code + expects and match the behavior of the subprocess module in Python 2 to avoid + introducing hard to track down bugs. + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). -- cgit v0.12 From 656c80809c77511ed33bec9c7f6a9c09860f0116 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 23 Mar 2013 23:35:06 +0200 Subject: Clean up references to threads in test_queue. --- Lib/test/test_queue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 86ad9c0..2cdfee4 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -46,6 +46,9 @@ class _TriggerThread(threading.Thread): class BlockingTestMixin: + def tearDown(self): + self.t = None + def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): self.t = _TriggerThread(trigger_func, trigger_args) self.t.start() @@ -260,7 +263,7 @@ class FailingQueue(queue.Queue): raise FailingQueueException("You Lose") return queue.Queue._get(self) -class FailingQueueTest(unittest.TestCase, BlockingTestMixin): +class FailingQueueTest(BlockingTestMixin, unittest.TestCase): def failing_queue_test(self, q): if q.qsize(): -- cgit v0.12 From 5be6d74a0d0ae111cd823d2b7a5896c77d8c8895 Mon Sep 17 00:00:00 2001 From: "Martin v. Loewis" Date: Sun, 24 Mar 2013 22:03:30 +0100 Subject: Issue #17425: Build with openssl 1.0.0k on Windows. --- Misc/NEWS | 2 ++ PC/VC6/readme.txt | 4 ++-- PC/VS8.0/pyproject.vsprops | 2 +- PCbuild/pyproject.vsprops | 2 +- PCbuild/readme.txt | 2 +- Tools/buildbot/external-common.bat | 4 ++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index e4d04b1..dbc12e7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1084,6 +1084,8 @@ Tests Build ----- +- Issue #17425: Build with openssl 1.0.0k on Windows. + - Issue #16754: Fix the incorrect shared library extension on linux. Introduce two makefile macros SHLIB_SUFFIX and EXT_SUFFIX. SO now has the value of SHLIB_SUFFIX again (as in 2.x and 3.1). The SO macro is removed in 3.4. diff --git a/PC/VC6/readme.txt b/PC/VC6/readme.txt index fa1ae7d..d93760d 100644 --- a/PC/VC6/readme.txt +++ b/PC/VC6/readme.txt @@ -153,9 +153,9 @@ _ssl Unpack into the "dist" directory, retaining the folder name from the archive - for example, the latest stable OpenSSL will install as - dist/openssl-1.0.0j + dist/openssl-1.0.0k - You need to use version 1.0.0j of OpenSSL. + You need to use version 1.0.0k of OpenSSL. You can install the NASM assembler from http://www.nasm.us/ diff --git a/PC/VS8.0/pyproject.vsprops b/PC/VS8.0/pyproject.vsprops index 2a82e49..b75b2c6 100644 --- a/PC/VS8.0/pyproject.vsprops +++ b/PC/VS8.0/pyproject.vsprops @@ -58,7 +58,7 @@ />