diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 75b5150f821b1c..f7c30eb6dd103a 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1309,14 +1309,6 @@ class SharedMemoryManager(BaseManager): _Server = SharedMemoryServer def __init__(self, *args, **kwargs): - if os.name == "posix": - # bpo-36867: Ensure the resource_tracker is running before - # launching the manager process, so that concurrent - # shared_memory manipulation both in the manager and in the - # current process does not create two resource_tracker - # processes. - from . import resource_tracker - resource_tracker.ensure_running() BaseManager.__init__(self, *args, **kwargs) util.debug(f"{self.__class__.__name__} created by pid {getpid()}") diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 61a6dd66e72e67..fa20bfd1aca6d1 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -39,7 +39,6 @@ _CLEANUP_FUNCS.update({ 'semaphore': _multiprocessing.sem_unlink, - 'shared_memory': _posixshmem.shm_unlink, }) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 184e36704baaeb..ebc88858762e8c 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -113,9 +113,6 @@ def __init__(self, name=None, create=False, size=0): self.unlink() raise - from .resource_tracker import register - register(self._name, "shared_memory") - else: # Windows Named Shared Memory @@ -234,9 +231,7 @@ def unlink(self): called once (and only once) across all processes which have access to the shared memory block.""" if _USE_POSIX and self._name: - from .resource_tracker import unregister _posixshmem.shm_unlink(self._name) - unregister(self._name, "shared_memory") _encoding = "utf8" diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c717d0aad2874b..0a5b93e49ffef5 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3829,6 +3829,27 @@ def test_shared_memory_across_processes(self): sms.close() + def test_shared_memory_across_independently_started_processes(self): + # A Process may not trigger the same exit path as an independently + # spawned process (for example, resource trackers on a separate + # process). This tests that independently created processes can + # indeed continue to access shared memory and that that shared + # memory persists at least so long as one of the processes with a + # handle on it is alive. + sms = shared_memory.SharedMemory('test04_indep', True, size=256) + self.addCleanup(sms.unlink) + + prog = ( + "from multiprocessing import shared_memory; " + "sms = shared_memory.SharedMemory('test04_indep'); " + "sms.buf[:4] = b'1234'; " + "sms.close()" + ) + rc, out, err = test.support.script_helper.assert_python_ok('-c', prog) + + self.assertFalse(err) + self.assertEqual(bytes(sms.buf[:4]), b'1234') + @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") def test_shared_memory_SharedMemoryServer_ignores_sigint(self): # bpo-36368: protect SharedMemoryManager server process from @@ -3853,27 +3874,6 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() - @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") - def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): - # bpo-36867: test that a SharedMemoryManager uses the - # same resource_tracker process as its parent. - cmd = '''if 1: - from multiprocessing.managers import SharedMemoryManager - - - smm = SharedMemoryManager() - smm.start() - sl = smm.ShareableList(range(10)) - smm.shutdown() - ''' - rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd) - - # Before bpo-36867 was fixed, a SharedMemoryManager not using the same - # resource_tracker process as its parent would make the parent's - # tracker complain about sl being leaked even though smm.shutdown() - # properly released sl. - self.assertFalse(err) - def test_shared_memory_SharedMemoryManager_basics(self): smm1 = multiprocessing.managers.SharedMemoryManager() with self.assertRaises(ValueError): @@ -4012,49 +4012,6 @@ def test_shared_memory_ShareableList_pickling(self): deserialized_sl.shm.close() sl.shm.close() - def test_shared_memory_cleaned_after_process_termination(self): - cmd = '''if 1: - import os, time, sys - from multiprocessing import shared_memory - - # Create a shared_memory segment, and send the segment name - sm = shared_memory.SharedMemory(create=True, size=10) - sys.stdout.write(sm.name + '\\n') - sys.stdout.flush() - time.sleep(100) - ''' - with subprocess.Popen([sys.executable, '-E', '-c', cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as p: - name = p.stdout.readline().strip().decode() - - # killing abruptly processes holding reference to a shared memory - # segment should not leak the given memory segment. - p.terminate() - p.wait() - - deadline = time.monotonic() + 60 - t = 0.1 - while time.monotonic() < deadline: - time.sleep(t) - t = min(t*2, 5) - try: - smm = shared_memory.SharedMemory(name, create=False) - except FileNotFoundError: - break - else: - raise AssertionError("A SharedMemory segment was leaked after" - " a process was abruptly terminated.") - - if os.name == 'posix': - # A warning was emitted by the subprocess' own - # resource_tracker (on Windows, shared memory segments - # are released automatically by the OS). - err = p.stderr.read().decode() - self.assertIn( - "resource_tracker: There appear to be 1 leaked " - "shared_memory objects to clean up at shutdown", err) - # # # @@ -5006,9 +4963,6 @@ def create_and_register_resource(rtype): if rtype == "semaphore": lock = mp.Lock() return lock, lock._semlock.name - elif rtype == "shared_memory": - sm = SharedMemory(create=True, size=10) - return sm, sm._name else: raise ValueError( "Resource type {{}} not understood".format(rtype)) diff --git a/Misc/NEWS.d/next/Library/2019-09-11-11-08-53.bpo-38119.da_hwK.rst b/Misc/NEWS.d/next/Library/2019-09-11-11-08-53.bpo-38119.da_hwK.rst new file mode 100644 index 00000000000000..5c29d4a2f36ca8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-11-11-08-53.bpo-38119.da_hwK.rst @@ -0,0 +1 @@ +Fix resource tracker treatment of shared memory as if it were a semaphore.