diff options
| -rw-r--r-- | Lib/multiprocessing/forkserver.py | 5 | ||||
| -rw-r--r-- | Lib/test/_test_multiprocessing.py | 34 | ||||
| -rw-r--r-- | Lib/test/libregrtest/refleak.py | 32 | ||||
| -rw-r--r-- | Lib/test/support/__init__.py | 33 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst | 4 |
5 files changed, 75 insertions, 33 deletions
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index b9f9b9d..69b842a 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -236,8 +236,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): code = 1 try: listener.close() + selector.close() + unused_fds = [alive_r, child_w, sig_r, sig_w] + unused_fds.extend(pid_to_fd.values()) code = _serve_one(child_r, fds, - (alive_r, child_w, sig_r, sig_w), + unused_fds, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 329a6d2..a14fa74 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -494,6 +494,40 @@ class _TestProcess(BaseTestCase): self.assertIs(wr(), None) self.assertEqual(q.get(), 5) + @classmethod + def _test_child_fd_inflation(self, evt, q): + q.put(test.support.fd_count()) + evt.wait() + + def test_child_fd_inflation(self): + # Number of fds in child processes should not grow with the + # number of running children. + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sm = multiprocessing.get_start_method() + if sm == 'fork': + # The fork method by design inherits all fds from the parent, + # trying to go against it is a lost battle + self.skipTest('test not appropriate for {}'.format(sm)) + + N = 5 + evt = self.Event() + q = self.Queue() + + procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q)) + for i in range(N)] + for p in procs: + p.start() + + try: + fd_counts = [q.get() for i in range(N)] + self.assertEqual(len(set(fd_counts)), 1, fd_counts) + + finally: + evt.set() + for p in procs: + p.join() # # diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 8e93816..efe5210 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -7,36 +7,6 @@ from inspect import isabstract from test import support -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except Exception: - MAXFD = 256 - - -def fd_count(): - """Count the number of open file descriptors""" - if sys.platform.startswith(('linux', 'freebsd')): - try: - names = os.listdir("/proc/self/fd") - return len(names) - except FileNotFoundError: - pass - - count = 0 - for fd in range(MAXFD): - try: - # Prefer dup() over fstat(). fstat() can require input/output - # whereas dup() doesn't. - fd2 = os.dup(fd) - except OSError as e: - if e.errno != errno.EBADF: - raise - else: - os.close(fd2) - count += 1 - return count - - def dash_R(the_module, test, indirect_test, huntrleaks): """Run a test multiple times, looking for reference leaks. @@ -174,7 +144,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): func1 = sys.getallocatedblocks func2 = sys.gettotalrefcount gc.collect() - return func1(), func2(), fd_count() + return func1(), func2(), support.fd_count() def clear_caches(): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 313c230..3a4d27e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -107,7 +107,7 @@ __all__ = [ "check_warnings", "check_no_resource_warning", "EnvironmentVarGuard", "run_with_locale", "swap_item", "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict", - "run_with_tz", "PGO", "missing_compiler_executable", + "run_with_tz", "PGO", "missing_compiler_executable", "fd_count", ] class Error(Exception): @@ -2647,3 +2647,34 @@ def disable_faulthandler(): finally: if is_enabled: faulthandler.enable(file=fd, all_threads=True) + + +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors. + """ + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count diff --git a/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst new file mode 100644 index 0000000..cff0f9a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-07-22-12-12-42.bpo-26732.lYLWBH.rst @@ -0,0 +1,4 @@ +Fix too many fds in processes started with the "forkserver" method. + +A child process would inherit as many fds as the number of still-running +children. |