From 83ac6e7ba5225274d991e212a810392e2060020c Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 19 Jul 2020 19:17:49 +0900 Subject: [PATCH 01/11] Add contextlib.aclosing() and update docs for explicit aclose() --- Doc/library/contextlib.rst | 32 ++++++++++++++++++++++++++++++++ Doc/reference/expressions.rst | 13 ++++++++++--- Lib/contextlib.py | 25 +++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 0aa4ad76523480..f0155ebe3d6fa7 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -154,6 +154,38 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. function:: aclosing(thing) + + Return an async context manager that calls the ``aclose()`` method of *thing* + upon completion of the block. This is basically equivalent to:: + + from contextlib import asynccontextmanager + + @asynccontextmanager + def aclosing(thing): + try: + yield thing + finally: + await thing.aclose() + + And lets you write code like this:: + + async def ticker(delay, to): + for i in range(to): + yield i + await asyncio.sleep(delay) + + with aclosing(ticker(10)) as ticks: + async for tick in ticks: + print(tick) + + without needing to explicitly call the ``aclose()`` method of ``ticks``. + Even if an error occurs, ``ticks.aclose()`` will be called when + the :keyword:`async with` block is exited. + + .. versionadded:: 3.10 + + .. _simplifying-support-for-single-optional-context-managers: .. function:: nullcontext(enter_result=None) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8036a491c29ab1..bf7e6a87154d2d 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -639,6 +639,13 @@ after resuming depends on the method which resumed the execution. If :meth:`~agen.asend` is used, then the result will be the value passed in to that method. +If an asynchronous generator is not resumed before it reaches the end of function +but the event loop where the generator is bound to continues to run (e.g., when +the caller task is cancelled), the caller must explicitly close the async +generator by calling :meth:`~agen:aclose` method to free the resources allocated +in the async generator function's stack and detach the generator from the event +loop. + In an asynchronous generator function, yield expressions are allowed anywhere in a :keyword:`try` construct. However, if an asynchronous generator is not resumed before it is finalized (by reaching a zero reference count or by @@ -650,9 +657,9 @@ generator-iterator's :meth:`~agen.aclose` method and run the resulting coroutine object, thus allowing any pending :keyword:`!finally` clauses to execute. -To take care of finalization, an event loop should define -a *finalizer* function which takes an asynchronous generator-iterator -and presumably calls :meth:`~agen.aclose` and executes the coroutine. +To take care of finalization upon event loop termination, an event loop should +define a *finalizer* function which takes an asynchronous generator-iterator and +presumably calls :meth:`~agen.aclose` and executes the coroutine. This *finalizer* may be registered by calling :func:`sys.set_asyncgen_hooks`. When first iterated over, an asynchronous generator-iterator will store the registered *finalizer* to be called upon finalization. For a reference example diff --git a/Lib/contextlib.py b/Lib/contextlib.py index ff92d9f913f4c2..37c66df7b4cc29 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -303,6 +303,31 @@ def __exit__(self, *exc_info): self.thing.close() +class aclosing(AbstractAsyncContextManager): + """Context to automatically aclose an async generator at the end of a block. + + Code like this: + + async with aclosing(.fetch()) as agen: + + + is equivalent to this: + + agen = .fetch() + try: + + finally: + await agen.aclose() + + """ + def __init__(self, thing): + self.thing = thing + async def __aenter__(self): + return self.thing + async def __aexit__(self, *exc_info): + await self.thing.aclose() + + class _RedirectStream(AbstractContextManager): _stream = None From a9b2d91ebd8e25869a8270448ed09f655db28091 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 19 Jul 2020 19:32:19 +0900 Subject: [PATCH 02/11] Add test case --- Lib/test/test_contextlib_async.py | 59 ++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 43fb7fced1bfdb..1e7eee28e96e96 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,5 +1,5 @@ import asyncio -from contextlib import asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack +from contextlib import asynccontextmanager, AbstractAsyncContextManager, aclosing, AsyncExitStack import functools from test import support import unittest @@ -279,6 +279,63 @@ async def woohoo(self, func, args, kwds): self.assertEqual(target, (11, 22, 33, 44)) +class AclosingTestCase(unittest.TestCase): + + @support.requires_docstrings + def test_instance_docs(self): + cm_docstring = aclosing.__doc__ + obj = aclosing(None) + self.assertEqual(obj.__doc__, cm_docstring) + + @_async_test + async def test_aclosing(self): + state = [] + class C: + async def aclose(self): + state.append(1) + x = C() + self.assertEqual(state, []) + async with aclosing(x) as y: + self.assertEqual(x, y) + self.assertEqual(state, [1]) + + @_async_test + async def test_aclosing_error(self): + state = [] + class C: + async def aclose(self): + state.append(1) + x = C() + self.assertEqual(state, []) + with self.assertRaises(ZeroDivisionError): + async with aclosing(x) as y: + self.assertEqual(x, y) + 1 / 0 + self.assertEqual(state, [1]) + + @_async_test + async def test_aclosing_bpo41229(self): + state = [] + + class Resource: + def __del__(self): + state.append(1) + + async def agenfunc(): + r = Resource() + yield -1 + yield -2 + + x = agenfunc() + self.assertEqual(state, []) + with self.assertRaises(ZeroDivisionError): + async with aclosing(x) as y: + self.assertEqual(x, y) + self.assertEqual(-1, await x.__anext__()) + 1 / 0 + self.assertEqual(state, [1]) + + class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase): class SyncAsyncExitStack(AsyncExitStack): @staticmethod From 25556e1f05db6de7cb7ed5e6aa2691bc2f23f561 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 19 Jul 2020 20:07:06 +0900 Subject: [PATCH 03/11] Fix typo --- Doc/reference/expressions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index bf7e6a87154d2d..0e07a925134108 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -642,7 +642,7 @@ that method. If an asynchronous generator is not resumed before it reaches the end of function but the event loop where the generator is bound to continues to run (e.g., when the caller task is cancelled), the caller must explicitly close the async -generator by calling :meth:`~agen:aclose` method to free the resources allocated +generator by calling :meth:`~agen.aclose` method to free the resources allocated in the async generator function's stack and detach the generator from the event loop. From 65faafe9468f80ced2bdf4f48501415df327eab0 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 19 Jul 2020 20:11:39 +0900 Subject: [PATCH 04/11] Add NEWS fragment --- .../next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst diff --git a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst new file mode 100644 index 00000000000000..d0cbeef8d91014 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst @@ -0,0 +1,2 @@ +Add ``contextlib.aclosing`` analogous to ``contextlib.closing`` but for +async generators and arbitrary objects with an ``aclose`` coroutine method. From aeae6332fc351a72c362ed32ba91015b23dfc225 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Fri, 2 Oct 2020 01:04:48 +0900 Subject: [PATCH 05/11] Apply aeros' review comments --- Doc/library/contextlib.rst | 2 +- Lib/contextlib.py | 2 +- Lib/test/test_contextlib_async.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index f0155ebe3d6fa7..beed6eaaf46d7e 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -154,7 +154,7 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. -.. function:: aclosing(thing) +.. class:: aclosing(thing) Return an async context manager that calls the ``aclose()`` method of *thing* upon completion of the block. This is basically equivalent to:: diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 37c66df7b4cc29..ea5807c121619f 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -304,7 +304,7 @@ def __exit__(self, *exc_info): class aclosing(AbstractAsyncContextManager): - """Context to automatically aclose an async generator at the end of a block. + """Async context manager for safely finalizing an async generator using its ``aclose()`` method. Code like this: diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 1e7eee28e96e96..3765f6cbf28c51 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,5 +1,5 @@ import asyncio -from contextlib import asynccontextmanager, AbstractAsyncContextManager, aclosing, AsyncExitStack +from contextlib import aclosing, asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack import functools from test import support import unittest From 53c881d9409772795eee225f1d37fdf764807dda Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sat, 17 Oct 2020 15:47:45 +0900 Subject: [PATCH 06/11] Fix missing async/await as reviewed by belm0 --- Doc/library/contextlib.rst | 4 ++-- Lib/contextlib.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index beed6eaaf46d7e..e8229e59754ce8 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -162,7 +162,7 @@ Functions and classes provided: from contextlib import asynccontextmanager @asynccontextmanager - def aclosing(thing): + async def aclosing(thing): try: yield thing finally: @@ -175,7 +175,7 @@ Functions and classes provided: yield i await asyncio.sleep(delay) - with aclosing(ticker(10)) as ticks: + async with aclosing(ticker(10)) as ticks: async for tick in ticks: print(tick) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index ea5807c121619f..2fa212ed512336 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -313,7 +313,7 @@ class aclosing(AbstractAsyncContextManager): is equivalent to this: - agen = .fetch() + agen = await .fetch() try: finally: From 351b85c0ff073885917ba97580f4eba32878ec36 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sat, 17 Oct 2020 15:56:24 +0900 Subject: [PATCH 07/11] Use better example from #22640 and update async-gen expr doc --- Doc/library/contextlib.rst | 23 ++++++++++++----------- Doc/reference/expressions.rst | 15 +++++++++------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index e8229e59754ce8..e42f5a93281663 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -168,20 +168,21 @@ Functions and classes provided: finally: await thing.aclose() - And lets you write code like this:: + Significantly, ``aclosing()`` supports deterministic cleanup of async + generators when they happen to exit early by :keyword:`break` or an + exception. For example:: - async def ticker(delay, to): - for i in range(to): - yield i - await asyncio.sleep(delay) + from contextlib import aclosing - async with aclosing(ticker(10)) as ticks: - async for tick in ticks: - print(tick) + async with aclosing(my_generator()) as values: + async for value in values: + if value == 42: + break - without needing to explicitly call the ``aclose()`` method of ``ticks``. - Even if an error occurs, ``ticks.aclose()`` will be called when - the :keyword:`async with` block is exited. + This pattern ensures that the generator's async exit code is executed in + the same context as its iterations (so that exceptions and context + variables work as expected, and the exit code isn't run after the + lifetime of some task it depends on). .. versionadded:: 3.10 diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 0e07a925134108..a41974719e5ffb 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -639,12 +639,15 @@ after resuming depends on the method which resumed the execution. If :meth:`~agen.asend` is used, then the result will be the value passed in to that method. -If an asynchronous generator is not resumed before it reaches the end of function -but the event loop where the generator is bound to continues to run (e.g., when -the caller task is cancelled), the caller must explicitly close the async -generator by calling :meth:`~agen.aclose` method to free the resources allocated -in the async generator function's stack and detach the generator from the event -loop. +If an asynchronous generator happens to exit early by :keyword:`break`, the caller +task being cancelled, or other exceptions, the generator's async cleanup code +will run and possibly raise exceptions or access context variables in an +unexpected context--perhaps after the lifetime of tasks it depends, or +during the event loop shutdown when the async-generator garbage collection hook +is called. +To prevent this, the caller must explicitly close the async generator by calling +:meth:`~agen.aclose` method to finalize the generator and ultimately detach it +from the event loop. In an asynchronous generator function, yield expressions are allowed anywhere in a :keyword:`try` construct. However, if an asynchronous generator is not From 69ce971f339fbabf1df644e9aba12189f9b8ded8 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sat, 17 Oct 2020 15:59:58 +0900 Subject: [PATCH 08/11] Mention the primary motivation for this PR --- .../next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst index d0cbeef8d91014..89413eda95e17d 100644 --- a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst +++ b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst @@ -1,2 +1,2 @@ -Add ``contextlib.aclosing`` analogous to ``contextlib.closing`` but for -async generators and arbitrary objects with an ``aclose`` coroutine method. +Add ``contextlib.aclosing`` for deterministic cleanup of async generators +which is analogous to ``contextlib.closing`` for non-async generators. From e2249a5e9f5f8ee63bacbfec429ad492530e09ad Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Oct 2020 00:12:07 +0900 Subject: [PATCH 09/11] Add contributor names in the changelog. --- .../NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst index 89413eda95e17d..926133221d4179 100644 --- a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst +++ b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst @@ -1,2 +1,3 @@ Add ``contextlib.aclosing`` for deterministic cleanup of async generators which is analogous to ``contextlib.closing`` for non-async generators. +Patch by Joongi Kim and John Belmonte. From 32e7c922df089b1cef542d667a4dcd384749f53b Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Oct 2020 00:17:30 +0900 Subject: [PATCH 10/11] Generalize aclosing docstring and revert mistakenly added await keyword --- Lib/contextlib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 2fa212ed512336..c1ca9980d8e4d1 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -304,7 +304,8 @@ def __exit__(self, *exc_info): class aclosing(AbstractAsyncContextManager): - """Async context manager for safely finalizing an async generator using its ``aclose()`` method. + """Async context manager for safely finalizing asynchronously cleaned-up + resources such as async generators, calling their ``aclose()`` method. Code like this: @@ -313,7 +314,7 @@ class aclosing(AbstractAsyncContextManager): is equivalent to this: - agen = await .fetch() + agen = .fetch() try: finally: From 6adce3dfb4250d731a1a907552dba36af5396de5 Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Sun, 18 Oct 2020 00:20:16 +0900 Subject: [PATCH 11/11] De-pluralize the docstring since aclosing applies to one object --- Lib/contextlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index c1ca9980d8e4d1..82ddc1497d8632 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -304,8 +304,8 @@ def __exit__(self, *exc_info): class aclosing(AbstractAsyncContextManager): - """Async context manager for safely finalizing asynchronously cleaned-up - resources such as async generators, calling their ``aclose()`` method. + """Async context manager for safely finalizing an asynchronously cleaned-up + resource such as an async generator, calling its ``aclose()`` method. Code like this: