From fce0416c1d10249d975f605809d074a6401b367a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 8 Oct 2022 06:43:21 +0000 Subject: [PATCH 1/4] deprecate watchers --- Doc/whatsnew/3.12.rst | 16 ++++++++++ Lib/asyncio/unix_events.py | 15 ++++++++++ Lib/test/test_asyncio/test_subprocess.py | 30 +++++++++++-------- ...2-10-08-06-59-46.gh-issue-94597.TsS0oT.rst | 1 + 4 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f873974b3e78fe..0c18fad5ddd78c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -102,6 +102,22 @@ New Modules Improved Modules ================ +asyncio +------- + +* On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default + if :func:`os.pidfd_open` is available instead of + :class:`~asyncio.ThreadedChildWatcher`. + (Contributed by Kumar Aditya in :gh:`98024`.) + +* :class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` + and :class:`~asyncio.SafeChildWatcher` child watchers are deprecated and + will be removed in Python 3.14. It is recommended to not manually + configure child watcher as the event loop now uses the best available + child watcher for the platform. + (Contributed by Kumar Aditya in :gh:`94597`.) + + pathlib ------- diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 7fc75cd17ef741..2edffd8d87f469 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1022,6 +1022,13 @@ class SafeChildWatcher(BaseChildWatcher): big number of children (O(n) each time SIGCHLD is raised) """ + def __init__(self): + super().__init__() + warnings._deprecated("SafeChildWatcher", + "{name!r} is deprecated since Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) + def close(self): self._callbacks.clear() super().close() @@ -1100,6 +1107,10 @@ def __init__(self): self._lock = threading.Lock() self._zombies = {} self._forks = 0 + warnings._deprecated("FastChildWatcher", + "{name!r} is deprecated since Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) def close(self): self._callbacks.clear() @@ -1212,6 +1223,10 @@ class MultiLoopChildWatcher(AbstractChildWatcher): def __init__(self): self._callbacks = {} self._saved_sighandler = None + warnings._deprecated("MultiLoopChildWatcher", + "{name!r} is deprecated since Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) def is_active(self): return self._saved_sighandler is not None diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 6ba889407b802e..3175e61c0cac84 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -688,7 +688,7 @@ def setUp(self): self.loop = policy.new_event_loop() self.set_event_loop(self.loop) - watcher = self.Watcher() + watcher = self._get_watcher() watcher.attach_loop(self.loop) policy.set_child_watcher(watcher) @@ -703,24 +703,28 @@ def tearDown(self): class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.ThreadedChildWatcher - - @unittest.skip("bpo-38323: MultiLoopChildWatcher has a race condition \ - and these tests can hang the test suite") - class SubprocessMultiLoopWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - Watcher = unix_events.MultiLoopChildWatcher + def _get_watcher(self): + return unix_events.ThreadedChildWatcher() class SubprocessSafeWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.SafeChildWatcher + def _get_watcher(self): + with self.assertWarns(DeprecationWarning): + return unix_events.SafeChildWatcher() + + class MultiLoopChildWatcherTests(test_utils.TestCase): + + def test_warns(self): + with self.assertWarns(DeprecationWarning): + unix_events.MultiLoopChildWatcher() class SubprocessFastWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.FastChildWatcher + def _get_watcher(self): + with self.assertWarns(DeprecationWarning): + return unix_events.FastChildWatcher() @unittest.skipUnless( _has_pidfd_support(), @@ -728,7 +732,9 @@ class SubprocessFastWatcherTests(SubprocessWatcherMixin, ) class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.PidfdChildWatcher + + def _get_watcher(self): + return unix_events.PidfdChildWatcher() class GenericWatcherTests(test_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst new file mode 100644 index 00000000000000..a5df4cb86e82b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst @@ -0,0 +1 @@ +:class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` and :class:`~asyncio.SafeChildWatcher` child watchers are deprecated and will be removed in Python 3.14. Patch by Kumar Aditya. From b6ac303d4e8a2d6cd362f3d35f0968f51a37548d Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 8 Oct 2022 11:22:14 +0000 Subject: [PATCH 2/4] fix tests --- Lib/test/test_asyncio/test_events.py | 10 +++++++--- Lib/test/test_asyncio/test_streams.py | 6 ++++-- Lib/test/test_asyncio/test_unix_events.py | 21 ++++++++++++++++----- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 80d7152128c469..98b55dec37f478 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -22,7 +22,7 @@ import unittest from unittest import mock import weakref - +import warnings if sys.platform not in ('win32', 'vxworks'): import tty @@ -2055,7 +2055,9 @@ def test_remove_fds_after_closing(self): class UnixEventLoopTestsMixin(EventLoopTestsMixin): def setUp(self): super().setUp() - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher) @@ -2652,7 +2654,9 @@ def setUp(self): asyncio.set_event_loop(self.loop) if sys.platform != 'win32': - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 0c49099bc499a5..8fb9313e09dd0e 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -9,6 +9,7 @@ import threading import unittest from unittest import mock +import warnings from test.support import socket_helper try: import ssl @@ -791,8 +792,9 @@ def test_read_all_from_pipe_reader(self): protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) transport, _ = self.loop.run_until_complete( self.loop.connect_read_pipe(lambda: protocol, pipe)) - - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) try: asyncio.set_child_watcher(watcher) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 03fb5e649d8e97..025da0f20ed47e 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -12,6 +12,7 @@ import threading import unittest from unittest import mock +import warnings from test.support import os_helper from test.support import socket_helper @@ -1686,12 +1687,16 @@ def test_close(self, m_waitpid): class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): - return asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return asyncio.SafeChildWatcher() class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): - return asyncio.FastChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return asyncio.FastChildWatcher() class PolicyTests(unittest.TestCase): @@ -1724,7 +1729,9 @@ def test_get_default_child_watcher(self): def test_get_child_watcher_after_set(self): policy = self.create_policy() - watcher = asyncio.FastChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + watcher = asyncio.FastChildWatcher() policy.set_child_watcher(watcher) self.assertIs(policy._watcher, watcher) @@ -1745,7 +1752,9 @@ def f(): policy.get_event_loop().close() policy = self.create_policy() - policy.set_child_watcher(asyncio.SafeChildWatcher()) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + policy.set_child_watcher(asyncio.SafeChildWatcher()) th = threading.Thread(target=f) th.start() @@ -1757,7 +1766,9 @@ def test_child_watcher_replace_mainloop_existing(self): # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + watcher = asyncio.SafeChildWatcher() policy.set_child_watcher(watcher) watcher.attach_loop(loop) From b2d2867e0dfcb5fc23d5b082e169ff2017648481 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 8 Oct 2022 22:37:03 +0530 Subject: [PATCH 3/4] Apply suggestions Guido Co-authored-by: Guido van Rossum --- Doc/whatsnew/3.12.rst | 12 +++++++----- Lib/asyncio/unix_events.py | 6 +++--- .../2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 0c18fad5ddd78c..e3350679b4697f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -106,15 +106,17 @@ asyncio ------- * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default - if :func:`os.pidfd_open` is available instead of + if :func:`os.pidfd_open` is available and functional instead of :class:`~asyncio.ThreadedChildWatcher`. (Contributed by Kumar Aditya in :gh:`98024`.) -* :class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` - and :class:`~asyncio.SafeChildWatcher` child watchers are deprecated and +* The child watcher classes :class:`~asyncio.MultiLoopChildWatcher`, + :class:`~asyncio.FastChildWatcher` and + :class:`~asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. It is recommended to not manually - configure child watcher as the event loop now uses the best available - child watcher for the platform. + configure a child watcher as the event loop now uses the best available + child watcher for each platform (:class:`~asyncio.PidfdChildWatcher` + if supported and :class:`~asyncio.ThreadedChildWatcher` otherwise). (Contributed by Kumar Aditya in :gh:`94597`.) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 2edffd8d87f469..bdffc032e318fd 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1025,7 +1025,7 @@ class SafeChildWatcher(BaseChildWatcher): def __init__(self): super().__init__() warnings._deprecated("SafeChildWatcher", - "{name!r} is deprecated since Python 3.12 and will be " + "{name!r} is deprecated as of Python 3.12 and will be " "removed in Python {remove}.", remove=(3, 14)) @@ -1108,7 +1108,7 @@ def __init__(self): self._zombies = {} self._forks = 0 warnings._deprecated("FastChildWatcher", - "{name!r} is deprecated since Python 3.12 and will be " + "{name!r} is deprecated as of Python 3.12 and will be " "removed in Python {remove}.", remove=(3, 14)) @@ -1224,7 +1224,7 @@ def __init__(self): self._callbacks = {} self._saved_sighandler = None warnings._deprecated("MultiLoopChildWatcher", - "{name!r} is deprecated since Python 3.12 and will be " + "{name!r} is deprecated as of Python 3.12 and will be " "removed in Python {remove}.", remove=(3, 14)) diff --git a/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst index a5df4cb86e82b9..f504ccf39ec948 100644 --- a/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst +++ b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst @@ -1 +1 @@ -:class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` and :class:`~asyncio.SafeChildWatcher` child watchers are deprecated and will be removed in Python 3.14. Patch by Kumar Aditya. +The child watcher classes :class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` and :class:`~asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. Patch by Kumar Aditya. From 68db9565320c42922b247bd90112c056311dc900 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 8 Oct 2022 17:09:57 +0000 Subject: [PATCH 4/4] refactor pidfd check --- Lib/test/test_asyncio/test_subprocess.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3175e61c0cac84..915ad5587f0a48 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -4,7 +4,6 @@ import sys import unittest import warnings -import functools from unittest import mock import asyncio @@ -31,19 +30,6 @@ 'sys.stdout.buffer.write(data)'))] -@functools.cache -def _has_pidfd_support(): - if not hasattr(os, 'pidfd_open'): - return False - - try: - os.close(os.pidfd_open(os.getpid())) - except OSError: - return False - - return True - - def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -727,7 +713,7 @@ def _get_watcher(self): return unix_events.FastChildWatcher() @unittest.skipUnless( - _has_pidfd_support(), + unix_events.can_use_pidfd(), "operating system does not support pidfds", ) class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, @@ -764,7 +750,7 @@ async def execute(): @unittest.skipUnless( - _has_pidfd_support(), + unix_events.can_use_pidfd(), "operating system does not support pidfds", ) def test_create_subprocess_with_pidfd(self):