diff options
| -rw-r--r-- | Lib/multiprocessing/managers.py | 9 | ||||
| -rw-r--r-- | Lib/test/_test_multiprocessing.py | 58 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst | 1 |
3 files changed, 54 insertions, 14 deletions
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 2bad636..5141522 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -21,6 +21,7 @@ import signal import array import queue import time +import os from os import getpid from traceback import format_exc @@ -1349,6 +1350,14 @@ if HAS_SHMEM: _Server = SharedMemoryServer def __init__(self, *args, **kwargs): + if os.name == "posix": + # bpo-36867: Ensure the resource_tracker is running before + # launching the manager process, so that concurrent + # shared_memory manipulation both in the manager and in the + # current process does not create two resource_tracker + # processes. + from . import resource_tracker + resource_tracker.ensure_running() BaseManager.__init__(self, *args, **kwargs) util.debug(f"{self.__class__.__name__} created by pid {getpid()}") diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a50293c..772c963 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -17,6 +17,7 @@ import array import socket import random import logging +import subprocess import struct import operator import pickle @@ -3765,6 +3766,27 @@ class _TestSharedMemory(BaseTestCase): smm.shutdown() + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): + # bpo-36867: test that a SharedMemoryManager uses the + # same resource_tracker process as its parent. + cmd = '''if 1: + from multiprocessing.managers import SharedMemoryManager + + + smm = SharedMemoryManager() + smm.start() + sl = smm.ShareableList(range(10)) + smm.shutdown() + ''' + rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd) + + # Before bpo-36867 was fixed, a SharedMemoryManager not using the same + # resource_tracker process as its parent would make the parent's + # tracker complain about sl being leaked even though smm.shutdown() + # properly released sl. + self.assertFalse(err) + def test_shared_memory_SharedMemoryManager_basics(self): smm1 = multiprocessing.managers.SharedMemoryManager() with self.assertRaises(ValueError): @@ -3904,8 +3926,6 @@ class _TestSharedMemory(BaseTestCase): sl.shm.close() def test_shared_memory_cleaned_after_process_termination(self): - import subprocess - from multiprocessing import shared_memory cmd = '''if 1: import os, time, sys from multiprocessing import shared_memory @@ -3916,18 +3936,29 @@ class _TestSharedMemory(BaseTestCase): sys.stdout.flush() time.sleep(100) ''' - p = subprocess.Popen([sys.executable, '-E', '-c', cmd], - stdout=subprocess.PIPE) - name = p.stdout.readline().strip().decode() + with subprocess.Popen([sys.executable, '-E', '-c', cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + name = p.stdout.readline().strip().decode() - # killing abruptly processes holding reference to a shared memory - # segment should not leak the given memory segment. - p.terminate() - p.wait() - time.sleep(1.0) # wait for the OS to collect the segment + # killing abruptly processes holding reference to a shared memory + # segment should not leak the given memory segment. + p.terminate() + p.wait() + time.sleep(1.0) # wait for the OS to collect the segment - with self.assertRaises(FileNotFoundError): - smm = shared_memory.SharedMemory(name, create=False) + # The shared memory file was deleted. + with self.assertRaises(FileNotFoundError): + smm = shared_memory.SharedMemory(name, create=False) + + if os.name == 'posix': + # A warning was emitted by the subprocess' own + # resource_tracker (on Windows, shared memory segments + # are released automatically by the OS). + err = p.stderr.read().decode() + self.assertIn( + "resource_tracker: There appear to be 1 leaked " + "shared_memory objects to clean up at shutdown", err) # # @@ -4560,7 +4591,7 @@ class TestFlags(unittest.TestCase): print(json.dumps(flags)) def test_flags(self): - import json, subprocess + import json # start child process using unusual flags prog = ('from test._test_multiprocessing import TestFlags; ' + 'TestFlags.run_in_child()') @@ -4866,7 +4897,6 @@ class TestResourceTracker(unittest.TestCase): # # Check that killing process does not leak named semaphores # - import subprocess cmd = '''if 1: import time, os, tempfile import multiprocessing as mp diff --git a/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst b/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst new file mode 100644 index 0000000..ce925d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-13-13-02-43.bpo-36867.Qh-6mX.rst @@ -0,0 +1 @@ +Fix a bug making a SharedMemoryManager instance and its parent process use two separate resource_tracker processes.
\ No newline at end of file |