From 4932e14542e10191b6b598307641f6b54c5da239 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Fri, 20 Jun 2014 01:37:53 -0700 Subject: issue 20091 - index entry for __main__ in runpy docs. --- Doc/library/runpy.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index ee9fbcf..a764696 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -28,6 +28,9 @@ The :mod:`runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) + .. index:: + module: __main__ + Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using the standard import mechanism (refer to :pep:`302` for details) and then executed in a @@ -87,6 +90,9 @@ The :mod:`runpy` module provides two functions: .. function:: run_path(file_path, init_globals=None, run_name=None) + .. index:: + module: __main__ + Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a -- cgit v0.12 From 0e6f52a2114d4fb3bf196e6a17e3b5daea3c0cdc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Jun 2014 17:34:15 +0200 Subject: asyncio, Tulip issue 105: in debug mode, log callbacks taking more than 100 ms to be executed. --- Lib/asyncio/base_events.py | 32 ++++++++++++++++++++++++++----- Lib/test/test_asyncio/test_base_events.py | 28 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 9350989..9f9067e 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -17,6 +17,7 @@ to modify the meaning of the API call itself. import collections import concurrent.futures import heapq +import inspect import logging import socket import subprocess @@ -37,6 +38,15 @@ __all__ = ['BaseEventLoop', 'Server'] _MAX_WORKERS = 5 +def _format_handle(handle): + cb = handle._callback + if inspect.ismethod(cb) and isinstance(cb.__self__, tasks.Task): + # format the task + return repr(cb.__self__) + else: + return str(handle) + + class _StopError(BaseException): """Raised to stop the event loop.""" @@ -128,6 +138,9 @@ class BaseEventLoop(events.AbstractEventLoop): self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None self._debug = False + # In debug mode, if the execution of a callback or a step of a task + # exceed this duration in seconds, the slow callback/task is logged. + self.slow_callback_duration = 0.1 def __repr__(self): return ('<%s running=%s closed=%s debug=%s>' @@ -823,16 +836,16 @@ class BaseEventLoop(events.AbstractEventLoop): if logger.isEnabledFor(logging.INFO): t0 = self.time() event_list = self._selector.select(timeout) - t1 = self.time() - if t1-t0 >= 1: + dt = self.time() - t0 + if dt >= 1: level = logging.INFO else: level = logging.DEBUG if timeout is not None: logger.log(level, 'poll %.3f took %.3f seconds', - timeout, t1-t0) + timeout, dt) else: - logger.log(level, 'poll took %.3f seconds', t1-t0) + logger.log(level, 'poll took %.3f seconds', dt) else: event_list = self._selector.select(timeout) self._process_events(event_list) @@ -855,7 +868,16 @@ class BaseEventLoop(events.AbstractEventLoop): ntodo = len(self._ready) for i in range(ntodo): handle = self._ready.popleft() - if not handle._cancelled: + if handle._cancelled: + continue + if self._debug: + t0 = self.time() + handle._run() + dt = self.time() - t0 + if dt >= self.slow_callback_duration: + logger.warning('Executing %s took %.3f seconds', + _format_handle(handle), dt) + else: handle._run() handle = None # Needed to break cycles when an exception occurs. diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 059b41c..352af48 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -969,6 +969,34 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): with self.assertRaises(TypeError): self.loop.run_in_executor(None, coroutine_function) + @mock.patch('asyncio.base_events.logger') + def test_log_slow_callbacks(self, m_logger): + def stop_loop_cb(loop): + loop.stop() + + @asyncio.coroutine + def stop_loop_coro(loop): + yield from () + loop.stop() + + asyncio.set_event_loop(self.loop) + self.loop.set_debug(True) + self.loop.slow_callback_duration = 0.0 + + # slow callback + self.loop.call_soon(stop_loop_cb, self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing Handle.*stop_loop_cb.* took .* seconds$") + + # slow task + asyncio.async(stop_loop_coro(self.loop), loop=self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing Task.*stop_loop_coro.* took .* seconds$") + if __name__ == '__main__': unittest.main() -- cgit v0.12 From fe928de955ec5043dd7c8b9388dbb0ccfabbdc80 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 20 Jun 2014 14:59:11 -0400 Subject: Issue #21768: fix type in test_pydoc, patch by Claudiu Popa. --- Lib/test/test_pydoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 542b433..8d85c14 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -746,7 +746,7 @@ class TestDescriptions(unittest.TestCase): try: pydoc.render_doc(name) except ImportError: - self.fail('finding the doc of {!r} failed'.format(o)) + self.fail('finding the doc of {!r} failed'.format(name)) for name in ('notbuiltins', 'strrr', 'strr.translate', 'str.trrrranslate', 'builtins.strrr', -- cgit v0.12 From af9eb9628d833c8096e4438a48655ed07996137a Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 20 Jun 2014 15:16:35 -0400 Subject: Issue #21768: fix NameError in test_pydescr. Patch by Claudiu Popa. --- Lib/test/test_descr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 8bb7d6a..26c273a 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1149,7 +1149,7 @@ order (MRO) for bases """ except (TypeError, UnicodeEncodeError): pass else: - raise TestFailed("[chr(128)] slots not caught") + self.fail("[chr(128)] slots not caught") # Test leaks class Counted(object): -- cgit v0.12 From 504f5c36ef2419cb0c56450c3a733499c4576456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Fri, 20 Jun 2014 22:37:35 +0100 Subject: Issue #21491: socketserver: Fix a race condition in child processes reaping. --- Lib/socketserver.py | 56 ++++++++++++++++++++++++++++------------------------- Misc/NEWS | 2 ++ 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 46ee7c5..2f395fa 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -523,35 +523,39 @@ class ForkingMixIn: def collect_children(self): """Internal routine to wait for children that have exited.""" - if self.active_children is None: return + if self.active_children is None: + return + + # If we're above the max number of children, wait and reap them until + # we go back below threshold. Note that we use waitpid(-1) below to be + # able to collect children in size() syscalls instead + # of size(): the downside is that this might reap children + # which we didn't spawn, which is why we only resort to this when we're + # above max_children. while len(self.active_children) >= self.max_children: - # XXX: This will wait for any child process, not just ones - # spawned by this library. This could confuse other - # libraries that expect to be able to wait for their own - # children. try: - pid, status = os.waitpid(0, 0) + pid, _ = os.waitpid(-1, 0) + self.active_children.discard(pid) + except InterruptedError: + pass + except ChildProcessError: + # we don't have any children, we're done + self.active_children.clear() except OSError: - pid = None - if pid not in self.active_children: continue - self.active_children.remove(pid) - - # XXX: This loop runs more system calls than it ought - # to. There should be a way to put the active_children into a - # process group and then use os.waitpid(-pgid) to wait for any - # of that set, but I couldn't find a way to allocate pgids - # that couldn't collide. - for child in self.active_children: + break + + # Now reap all defunct children. + for pid in self.active_children.copy(): try: - pid, status = os.waitpid(child, os.WNOHANG) + pid, _ = os.waitpid(pid, os.WNOHANG) + # if the child hasn't exited yet, pid will be 0 and ignored by + # discard() below + self.active_children.discard(pid) + except ChildProcessError: + # someone else reaped it + self.active_children.discard(pid) except OSError: - pid = None - if not pid: continue - try: - self.active_children.remove(pid) - except ValueError as e: - raise ValueError('%s. x=%d and list=%r' % (e.message, pid, - self.active_children)) + pass def handle_timeout(self): """Wait for zombies after self.timeout seconds of inactivity. @@ -573,8 +577,8 @@ class ForkingMixIn: if pid: # Parent process if self.active_children is None: - self.active_children = [] - self.active_children.append(pid) + self.active_children = set() + self.active_children.add(pid) self.close_request(request) return else: diff --git a/Misc/NEWS b/Misc/NEWS index e6c9469..b8e7219 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,8 @@ Core and Builtins Library ------- +- Issue #21491: socketserver: Fix a race condition in child processes reaping. + - Issue #21722: The distutils "upload" command now exits with a non-zero return code when uploading fails. Patch by Martin Dengler. -- cgit v0.12