From d628fc1bf59fcf1e82a4a6aa912c0cbcf8f4381f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:29:51 +0100 Subject: [PATCH 01/26] Update popen_spawn_posix.py --- Lib/multiprocessing/popen_spawn_posix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 24b8634523e5f2..28c8000d92bc90 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -65,7 +65,7 @@ def _launch(self, process_obj): for fd in (parent_r, parent_w): if fd is not None: fds_to_close.append(fd) - self.finalizer = util.Finalize(self, util.close_fds, fds_to_close) + self.finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key for fd in (child_r, child_w): if fd is not None: From e369d5e554b5b3816f0a8543ae66275b61936912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:34:22 +0100 Subject: [PATCH 02/26] Update popen_fork.py --- Lib/multiprocessing/popen_fork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..f42a6359f146ad 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -75,9 +75,9 @@ def _launch(self, process_obj): os.close(child_w) os.close(child_r) self.finalizer = util.Finalize(self, util.close_fds, - (parent_r, parent_w,)) + (parent_r, parent_w,))._key self.sentinel = parent_r def close(self): if self.finalizer is not None: - self.finalizer() + util._finalizer_registry[self.finalizer()]() From 590f44643e50a1a7eaa2f9a2321a7fd3880580de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:35:07 +0100 Subject: [PATCH 03/26] Update popen_forkserver.py --- Lib/multiprocessing/popen_forkserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_forkserver.py b/Lib/multiprocessing/popen_forkserver.py index a56eb9bf11080b..ed1eca97970779 100644 --- a/Lib/multiprocessing/popen_forkserver.py +++ b/Lib/multiprocessing/popen_forkserver.py @@ -53,7 +53,7 @@ def _launch(self, process_obj): # parent process used by the child process. _parent_w = os.dup(w) self.finalizer = util.Finalize(self, util.close_fds, - (_parent_w, self.sentinel)) + (_parent_w, self.sentinel))._key with open(w, 'wb', closefd=True) as f: f.write(buf.getbuffer()) self.pid = forkserver.read_signed(self.sentinel) From 941e4c074aae3a3e89476e987c46d2222690fca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:36:37 +0100 Subject: [PATCH 04/26] Update popen_spawn_win32.py --- Lib/multiprocessing/popen_spawn_win32.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 9c4098d0fa4f1e..a30983b7314cdf 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -84,7 +84,7 @@ def __init__(self, process_obj): self._handle = hp self.sentinel = int(hp) self.finalizer = util.Finalize(self, _close_handles, - (self.sentinel, int(rhandle))) + (self.sentinel, int(rhandle)))._key # send information to child set_spawning_popen(self) @@ -128,4 +128,4 @@ def terminate(self): kill = terminate def close(self): - self.finalizer() + util._finalizer_registry[self.finalizer()]() From e2a93dcd6ead74542c17744d38fc8862e255d71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:48:10 +0100 Subject: [PATCH 05/26] Update managers.py --- Lib/multiprocessing/managers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index d97381926d47bc..f24a42b5e613c2 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -568,12 +568,15 @@ def start(self, initializer=None, initargs=()): # register a finalizer self._state.value = State.STARTED - self.shutdown = util.Finalize( + self._finalizer = util.Finalize( self, type(self)._finalize_manager, args=(self._process, self._address, self._authkey, self._state, self._Client), exitpriority=0 - ) + )._key + + def shutdown(self): + util._finalizer_registry[self._finalizer]() @classmethod def _run_server(cls, registry, address, authkey, serializer, writer, From 2513628df54796444a15f77f4d61d8d2b929cd46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:49:07 +0100 Subject: [PATCH 06/26] Update popen_spawn_win32.py --- Lib/multiprocessing/popen_spawn_win32.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index a30983b7314cdf..6e711c51a38cbd 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -83,7 +83,7 @@ def __init__(self, process_obj): self.returncode = None self._handle = hp self.sentinel = int(hp) - self.finalizer = util.Finalize(self, _close_handles, + self._finalizer = util.Finalize(self, _close_handles, (self.sentinel, int(rhandle)))._key # send information to child @@ -128,4 +128,4 @@ def terminate(self): kill = terminate def close(self): - util._finalizer_registry[self.finalizer()]() + util._finalizer_registry[self._finalizer]() From c55302bab84003e8734deec4144bfc0f2ab81335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:50:05 +0100 Subject: [PATCH 07/26] Update popen_fork.py --- Lib/multiprocessing/popen_fork.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index f42a6359f146ad..432d126db83ee5 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -74,10 +74,10 @@ def _launch(self, process_obj): else: os.close(child_w) os.close(child_r) - self.finalizer = util.Finalize(self, util.close_fds, - (parent_r, parent_w,))._key + self._finalizer = util.Finalize(self, util.close_fds, + (parent_r, parent_w,))._key self.sentinel = parent_r def close(self): if self.finalizer is not None: - util._finalizer_registry[self.finalizer()]() + util._finalizer_registry[self._finalizer]() From 409349f53f44595c33b6a1cccf08491523806b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:50:34 +0100 Subject: [PATCH 08/26] Update popen_forkserver.py --- Lib/multiprocessing/popen_forkserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_forkserver.py b/Lib/multiprocessing/popen_forkserver.py index ed1eca97970779..f0a84fa77a5971 100644 --- a/Lib/multiprocessing/popen_forkserver.py +++ b/Lib/multiprocessing/popen_forkserver.py @@ -52,8 +52,8 @@ def _launch(self, process_obj): # Keep a duplicate of the data pipe's write end as a sentinel of the # parent process used by the child process. _parent_w = os.dup(w) - self.finalizer = util.Finalize(self, util.close_fds, - (_parent_w, self.sentinel))._key + self._finalizer = util.Finalize(self, util.close_fds, + (_parent_w, self.sentinel))._key with open(w, 'wb', closefd=True) as f: f.write(buf.getbuffer()) self.pid = forkserver.read_signed(self.sentinel) From 2eeeeb6e4c7be488aeea9bcec16eed450a01c4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 5 Mar 2022 23:51:35 +0100 Subject: [PATCH 09/26] Update popen_spawn_posix.py --- Lib/multiprocessing/popen_spawn_posix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 28c8000d92bc90..0c1b6f23d54f4a 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -65,7 +65,7 @@ def _launch(self, process_obj): for fd in (parent_r, parent_w): if fd is not None: fds_to_close.append(fd) - self.finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key + self._finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key for fd in (child_r, child_w): if fd is not None: From ade0b2adb83c5e6a1c5d2306bb8c373bd794afbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 00:57:25 +0100 Subject: [PATCH 10/26] Add unit tests --- Lib/test/_test_multiprocessing.py | 36 ++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index bb73d9e7cc75e4..953f4672b9a673 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -807,6 +807,22 @@ def test_forkserver_sigkill(self): if os.name != 'nt': self.check_forkserver_death(signal.SIGKILL) + def test_process_pickling(self): + if self.TYPE == 'threads': + self.skipTest(f'test not appropriate for {self.TYPE}') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + proc = self.Process() + proc.start() + # Calling set_spawning_popen with a value other than None + # allows pickling the authentication keys of processes + # (.authkey), which is prevented by default in + # AuthenticationString for security reasons. + multiprocessing.context.set_spawning_popen(0) + serialized_proc = pickle.dumps(proc, protocol=proto) + deserialized_proc = pickle.loads(serialized_proc) + self.assertIsInstance(deserialized_proc, self.Process) + # # @@ -2920,7 +2936,25 @@ def test_mymanager_context_prestarted(self): manager.start() with manager: self.common(manager) - self.assertEqual(manager._process.exitcode, 0) + # bpo-30356: BaseManager._finalize_manager() sends SIGTERM + # to the manager process if it takes longer than 1 second to stop, + # which happens on slow buildbots. + self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + + def test_mymanager_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + manager = MyManager() + manager.start() + # Calling set_spawning_popen with a value other than None + # allows pickling the authentication keys of processes + # (.authkey), which is prevented by default in + # AuthenticationString for security reasons. + multiprocessing.context.set_spawning_popen(0) + serialized_manager = pickle.dumps(manager, protocol=proto) + deserialized_manager = pickle.loads(serialized_manager) + self.assertIsInstance(deserialized_manager, MyManager) + manager.shutdown() def common(self, manager): foo = manager.Foo() From 71eb4e15d314a0885b7add448b7fd5d85d7199cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 11:36:29 +0100 Subject: [PATCH 11/26] Add a NEWS entry --- .../next/Library/2022-11-30-11-25-08.bpo-46934.zk8HaW.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-11-30-11-25-08.bpo-46934.zk8HaW.rst diff --git a/Misc/NEWS.d/next/Library/2022-11-30-11-25-08.bpo-46934.zk8HaW.rst b/Misc/NEWS.d/next/Library/2022-11-30-11-25-08.bpo-46934.zk8HaW.rst new file mode 100644 index 00000000000000..a9387390b0d5cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-30-11-25-08.bpo-46934.zk8HaW.rst @@ -0,0 +1,3 @@ +Make started :class:`multiprocessing.Process` instances and started +:class:`multiprocessing.managers.BaseManager` instances serialisable. Patch by +Géry Ogam. From 1419e0d95803e9c890ed6b93fc60a79a0584e117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 12:57:49 +0100 Subject: [PATCH 12/26] Make State instances pickable with the old text protocol --- Lib/multiprocessing/managers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index f24a42b5e613c2..25185ae6fd2416 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -476,6 +476,12 @@ class State(object): STARTED = 1 SHUTDOWN = 2 + def __getstate__(self): + return self.value + + def __setstate__(self, state): + self.value = state + # # Mapping from serializer name to Listener and Client types # From 1e5363173be9103e0e45d9fcdc88350b49c0db94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 15:05:17 +0100 Subject: [PATCH 13/26] Add restoration of spawning_popen --- Lib/test/_test_multiprocessing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 953f4672b9a673..49841cd874f166 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -820,6 +820,7 @@ def test_process_pickling(self): # AuthenticationString for security reasons. multiprocessing.context.set_spawning_popen(0) serialized_proc = pickle.dumps(proc, protocol=proto) + multiprocessing.context.set_spawning_popen(None) deserialized_proc = pickle.loads(serialized_proc) self.assertIsInstance(deserialized_proc, self.Process) @@ -2952,6 +2953,7 @@ def test_mymanager_pickling(self): # AuthenticationString for security reasons. multiprocessing.context.set_spawning_popen(0) serialized_manager = pickle.dumps(manager, protocol=proto) + multiprocessing.context.set_spawning_popen(None) deserialized_manager = pickle.loads(serialized_manager) self.assertIsInstance(deserialized_manager, MyManager) manager.shutdown() From 56f0bb8d9891609c433a33eb240691e72ddc828c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:09 +0100 Subject: [PATCH 14/26] Revert "Update popen_spawn_posix.py" This reverts commit 2eeeeb6e4c7be488aeea9bcec16eed450a01c4b7. --- Lib/multiprocessing/popen_spawn_posix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 0c1b6f23d54f4a..28c8000d92bc90 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -65,7 +65,7 @@ def _launch(self, process_obj): for fd in (parent_r, parent_w): if fd is not None: fds_to_close.append(fd) - self._finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key + self.finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key for fd in (child_r, child_w): if fd is not None: From 9f761088c709e5a46986e97e247f06d1e89ceeec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:21 +0100 Subject: [PATCH 15/26] Revert "Update popen_forkserver.py" This reverts commit 409349f53f44595c33b6a1cccf08491523806b3d. --- Lib/multiprocessing/popen_forkserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_forkserver.py b/Lib/multiprocessing/popen_forkserver.py index f0a84fa77a5971..ed1eca97970779 100644 --- a/Lib/multiprocessing/popen_forkserver.py +++ b/Lib/multiprocessing/popen_forkserver.py @@ -52,8 +52,8 @@ def _launch(self, process_obj): # Keep a duplicate of the data pipe's write end as a sentinel of the # parent process used by the child process. _parent_w = os.dup(w) - self._finalizer = util.Finalize(self, util.close_fds, - (_parent_w, self.sentinel))._key + self.finalizer = util.Finalize(self, util.close_fds, + (_parent_w, self.sentinel))._key with open(w, 'wb', closefd=True) as f: f.write(buf.getbuffer()) self.pid = forkserver.read_signed(self.sentinel) From 5c7719d7bd10042be562b6e2c8d8d6f6b226baa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:23 +0100 Subject: [PATCH 16/26] Revert "Update popen_fork.py" This reverts commit c55302bab84003e8734deec4144bfc0f2ab81335. --- Lib/multiprocessing/popen_fork.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 432d126db83ee5..f42a6359f146ad 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -74,10 +74,10 @@ def _launch(self, process_obj): else: os.close(child_w) os.close(child_r) - self._finalizer = util.Finalize(self, util.close_fds, - (parent_r, parent_w,))._key + self.finalizer = util.Finalize(self, util.close_fds, + (parent_r, parent_w,))._key self.sentinel = parent_r def close(self): if self.finalizer is not None: - util._finalizer_registry[self._finalizer]() + util._finalizer_registry[self.finalizer()]() From 24b53f1421d9a81e8fa5ef0a29c150bcb6161c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:25 +0100 Subject: [PATCH 17/26] Revert "Update popen_spawn_win32.py" This reverts commit 2513628df54796444a15f77f4d61d8d2b929cd46. --- Lib/multiprocessing/popen_spawn_win32.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 6e711c51a38cbd..a30983b7314cdf 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -83,7 +83,7 @@ def __init__(self, process_obj): self.returncode = None self._handle = hp self.sentinel = int(hp) - self._finalizer = util.Finalize(self, _close_handles, + self.finalizer = util.Finalize(self, _close_handles, (self.sentinel, int(rhandle)))._key # send information to child @@ -128,4 +128,4 @@ def terminate(self): kill = terminate def close(self): - util._finalizer_registry[self._finalizer]() + util._finalizer_registry[self.finalizer()]() From 04eac3ce5dcf50f3ec24272fad64afa91dbb4353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:26 +0100 Subject: [PATCH 18/26] Revert "Update managers.py" This reverts commit e2a93dcd6ead74542c17744d38fc8862e255d71b. --- Lib/multiprocessing/managers.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 25185ae6fd2416..da6c7cd2b213c7 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -574,15 +574,12 @@ def start(self, initializer=None, initargs=()): # register a finalizer self._state.value = State.STARTED - self._finalizer = util.Finalize( + self.shutdown = util.Finalize( self, type(self)._finalize_manager, args=(self._process, self._address, self._authkey, self._state, self._Client), exitpriority=0 - )._key - - def shutdown(self): - util._finalizer_registry[self._finalizer]() + ) @classmethod def _run_server(cls, registry, address, authkey, serializer, writer, From 9638a367f59fcd5ba5b6fc44d62b10f6b15313a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:27 +0100 Subject: [PATCH 19/26] Revert "Update popen_spawn_win32.py" This reverts commit 941e4c074aae3a3e89476e987c46d2222690fca0. --- Lib/multiprocessing/popen_spawn_win32.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index a30983b7314cdf..9c4098d0fa4f1e 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -84,7 +84,7 @@ def __init__(self, process_obj): self._handle = hp self.sentinel = int(hp) self.finalizer = util.Finalize(self, _close_handles, - (self.sentinel, int(rhandle)))._key + (self.sentinel, int(rhandle))) # send information to child set_spawning_popen(self) @@ -128,4 +128,4 @@ def terminate(self): kill = terminate def close(self): - util._finalizer_registry[self.finalizer()]() + self.finalizer() From 242d0270e19d5e01493ad67cb4d64f142c8b9c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:28 +0100 Subject: [PATCH 20/26] Revert "Update popen_forkserver.py" This reverts commit 590f44643e50a1a7eaa2f9a2321a7fd3880580de. --- Lib/multiprocessing/popen_forkserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_forkserver.py b/Lib/multiprocessing/popen_forkserver.py index ed1eca97970779..a56eb9bf11080b 100644 --- a/Lib/multiprocessing/popen_forkserver.py +++ b/Lib/multiprocessing/popen_forkserver.py @@ -53,7 +53,7 @@ def _launch(self, process_obj): # parent process used by the child process. _parent_w = os.dup(w) self.finalizer = util.Finalize(self, util.close_fds, - (_parent_w, self.sentinel))._key + (_parent_w, self.sentinel)) with open(w, 'wb', closefd=True) as f: f.write(buf.getbuffer()) self.pid = forkserver.read_signed(self.sentinel) From dbbbe1b0dd80c5a74b51627d1a5db629623967cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:29 +0100 Subject: [PATCH 21/26] Revert "Update popen_fork.py" This reverts commit e369d5e554b5b3816f0a8543ae66275b61936912. --- Lib/multiprocessing/popen_fork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index f42a6359f146ad..625981cf47627c 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -75,9 +75,9 @@ def _launch(self, process_obj): os.close(child_w) os.close(child_r) self.finalizer = util.Finalize(self, util.close_fds, - (parent_r, parent_w,))._key + (parent_r, parent_w,)) self.sentinel = parent_r def close(self): if self.finalizer is not None: - util._finalizer_registry[self.finalizer()]() + self.finalizer() From 88056fe4676e0504ebb4a170426f88657de4cbbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:06:31 +0100 Subject: [PATCH 22/26] Revert "Update popen_spawn_posix.py" This reverts commit d628fc1bf59fcf1e82a4a6aa912c0cbcf8f4381f. --- Lib/multiprocessing/popen_spawn_posix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 28c8000d92bc90..24b8634523e5f2 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -65,7 +65,7 @@ def _launch(self, process_obj): for fd in (parent_r, parent_w): if fd is not None: fds_to_close.append(fd) - self.finalizer = util.Finalize(self, util.close_fds, fds_to_close)._key + self.finalizer = util.Finalize(self, util.close_fds, fds_to_close) for fd in (child_r, child_w): if fd is not None: From f055d118f24089513c4f0ca700ab6c45674d7287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 17:29:49 +0100 Subject: [PATCH 23/26] Use a simpler implementation of serialisability based on __getstate__/__setstate__ --- Lib/multiprocessing/managers.py | 9 +++++++++ Lib/multiprocessing/process.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index da6c7cd2b213c7..0466e6121942b8 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -737,6 +737,15 @@ def temp(self, /, *args, **kwds): temp.__name__ = typeid setattr(cls, typeid, temp) + def __getstate__(self): + state = vars(self).copy() + state['_finalizer'] = state['_finalizer']._key + return state + + def __setstate__(self, state): + vars(self).update(state) + self._finalizer = util._finalizer_registry[self._finalizer] + # # Subclass of set which get cleared after a fork # diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 3917d2e4fa63ec..7c7fcea0e52eb8 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -285,6 +285,16 @@ def __repr__(self): info.append('daemon') return '<%s>' % ' '.join(info) + def __getstate__(self): + state = vars(self).copy() + state['_finalizer'] = state['_finalizer']._key + return state + + def __setstate__(self, state): + from . import util + vars(self).update(state) + self._finalizer = util._finalizer_registry[self._finalizer] + ## def _bootstrap(self, parent_sentinel=None): From 3c6f5dabe14a2bd320d9a8e2fd672bc0724bb2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 18:21:59 +0100 Subject: [PATCH 24/26] Restore finalizer attribute names --- Lib/multiprocessing/managers.py | 4 ++-- Lib/multiprocessing/process.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 0466e6121942b8..84294143dc012e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -739,12 +739,12 @@ def temp(self, /, *args, **kwds): def __getstate__(self): state = vars(self).copy() - state['_finalizer'] = state['_finalizer']._key + state['shutdown'] = state['shutdown']._key return state def __setstate__(self, state): vars(self).update(state) - self._finalizer = util._finalizer_registry[self._finalizer] + self.shutdown = util._finalizer_registry[self.shutdown] # # Subclass of set which get cleared after a fork diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 7c7fcea0e52eb8..56a6ec54e67dc5 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -287,13 +287,13 @@ def __repr__(self): def __getstate__(self): state = vars(self).copy() - state['_finalizer'] = state['_finalizer']._key + state['finalizer'] = state['finalizer']._key return state def __setstate__(self, state): from . import util vars(self).update(state) - self._finalizer = util._finalizer_registry[self._finalizer] + self.finalizer = util._finalizer_registry[self.finalizer] ## From fe89f2daf7ab99aa26f0fafe1472ebecca2ceda2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 18:44:50 +0100 Subject: [PATCH 25/26] Move __getstate__/__setstate__ to the right Popen classes --- Lib/multiprocessing/popen_fork.py | 9 +++++++++ Lib/multiprocessing/popen_spawn_win32.py | 10 ++++++++++ Lib/multiprocessing/process.py | 10 ---------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627c..dc0c2eb078b16d 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -81,3 +81,12 @@ def _launch(self, process_obj): def close(self): if self.finalizer is not None: self.finalizer() + + def __getstate__(self): + state = vars(self).copy() + state['finalizer'] = state['finalizer']._key + return state + + def __setstate__(self, state): + vars(self).update(state) + self.finalizer = util._finalizer_registry[self.finalizer] diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 9c4098d0fa4f1e..0ea545af60f52e 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -129,3 +129,13 @@ def terminate(self): def close(self): self.finalizer() + + def __getstate__(self): + state = vars(self).copy() + state['finalizer'] = state['finalizer']._key + return state + + def __setstate__(self, state): + vars(self).update(state) + self.finalizer = util._finalizer_registry[self.finalizer] + diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 56a6ec54e67dc5..3917d2e4fa63ec 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -285,16 +285,6 @@ def __repr__(self): info.append('daemon') return '<%s>' % ' '.join(info) - def __getstate__(self): - state = vars(self).copy() - state['finalizer'] = state['finalizer']._key - return state - - def __setstate__(self, state): - from . import util - vars(self).update(state) - self.finalizer = util._finalizer_registry[self.finalizer] - ## def _bootstrap(self, parent_sentinel=None): From c2ab61e80c26bf86fb801e50e38cc115d17997d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Wed, 30 Nov 2022 18:47:50 +0100 Subject: [PATCH 26/26] Remove an empty line introduced by mistake in the last commit --- Lib/multiprocessing/popen_spawn_win32.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 0ea545af60f52e..b060af2a2984a5 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -138,4 +138,3 @@ def __getstate__(self): def __setstate__(self, state): vars(self).update(state) self.finalizer = util._finalizer_registry[self.finalizer] -