From 61f10d093d29d7dda3142bb5f5c4aa416b4ad903 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 01:43:47 +0200 Subject: [PATCH 01/27] Implement Runner class --- Lib/asyncio/runners.py | 62 +++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 9a5e9a48479ef7..fca3a64ffe23ca 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -1,10 +1,47 @@ -__all__ = 'run', +__all__ = ('Runner', 'run') from . import coroutines from . import events from . import tasks +class Runner: + def __init__(self, debug=None): + self._debug = debug + self._loop = None + + def __enter__(self): + if events._get_running_loop() is not None: + raise RuntimeError( + "asyncio.Runner cannot be called from a running event loop") + + self._loop = events.new_event_loop() + if self._debug is not None: + self._loop.set_debug(self._debug) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + _cancel_all_tasks(self._loop) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.run_until_complete(self._loop.shutdown_default_executor()) + finally: + self._loop.close() + self._loop = None + + def run(self, coro): + if self._loop is None: + raise RuntimeError( + "Runner.run() cannot be called outside of " + "'with Runner(): ...' context manager" + ) + if not coroutines.iscoroutine(coro): + raise ValueError("a coroutine was expected, got {!r}".format(coro)) + + return self._loop.run_until_complete(coro) + + + def run(main, *, debug=None): """Execute the coroutine and return the result. @@ -29,27 +66,8 @@ async def main(): asyncio.run(main()) """ - if events._get_running_loop() is not None: - raise RuntimeError( - "asyncio.run() cannot be called from a running event loop") - - if not coroutines.iscoroutine(main): - raise ValueError("a coroutine was expected, got {!r}".format(main)) - - loop = events.new_event_loop() - try: - events.set_event_loop(loop) - if debug is not None: - loop.set_debug(debug) - return loop.run_until_complete(main) - finally: - try: - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.run_until_complete(loop.shutdown_default_executor()) - finally: - events.set_event_loop(None) - loop.close() + with Runner(debug) as runner: + return runner.run(main) def _cancel_all_tasks(loop): From d47179965d48429a9fec57a78d5c5edacaeca94a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 15:00:30 +0200 Subject: [PATCH 02/27] Add get_loop() / new_loop() methods --- Lib/asyncio/runners.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index fca3a64ffe23ca..3a280b09346296 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -6,18 +6,12 @@ class Runner: - def __init__(self, debug=None): - self._debug = debug - self._loop = None + def __init__(self, *, debug=None): + self._loop = self.new_loop() + if debug is not None: + self._loop.set_debug(debug) def __enter__(self): - if events._get_running_loop() is not None: - raise RuntimeError( - "asyncio.Runner cannot be called from a running event loop") - - self._loop = events.new_event_loop() - if self._debug is not None: - self._loop.set_debug(self._debug) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -30,6 +24,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): self._loop = None def run(self, coro): + if events._get_running_loop() is not None: + raise RuntimeError( + "Runner.run() cannot be called from a running event loop") + if self._loop is None: raise RuntimeError( "Runner.run() cannot be called outside of " @@ -40,6 +38,12 @@ def run(self, coro): return self._loop.run_until_complete(coro) + def get_loop(self): + return self._loop + + def new_loop(self): + return events.new_event_loop() + def run(main, *, debug=None): From c68ba8506018a6f33be713229ad2f60009066f67 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 16:41:10 +0200 Subject: [PATCH 03/27] Clarify --- Lib/asyncio/runners.py | 53 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 3a280b09346296..0b39688a3d918c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -6,8 +6,32 @@ class Runner: - def __init__(self, *, debug=None): - self._loop = self.new_loop() + """A context manager that controls event loop life cycle. + + The context manager always creates a new event loop, allows to run async funtions + inside it, and properly finalizes the loop at the context manager exit. + + If debug is True, the event loop will be run in debug mode. + If factory is passed, it is used for new event loop creation. + + asyncio.run(main(), debug=True) + + is a shortcut for + + with asyncio.Runner(debug=True) as runner: + runner.run(main()) + + + .run() method can be called multiple times. + + This can be useful for interactive console (e.g. IPython), + unittest runners, console tools, -- everywhere when async code + is called from existing sync framework and where the preferred single + asyncio.run() call doesn't work. + + """ + def __init__(self, *, debug=None, factory=events.new_event_loop): + self._loop = factory() if debug is not None: self._loop.set_debug(debug) @@ -15,6 +39,10 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + """Shutdown and close event loop.""" try: _cancel_all_tasks(self._loop) self._loop.run_until_complete(self._loop.shutdown_asyncgens()) @@ -24,26 +52,16 @@ def __exit__(self, exc_type, exc_val, exc_tb): self._loop = None def run(self, coro): - if events._get_running_loop() is not None: - raise RuntimeError( - "Runner.run() cannot be called from a running event loop") - - if self._loop is None: - raise RuntimeError( - "Runner.run() cannot be called outside of " - "'with Runner(): ...' context manager" - ) + """Run a coroutine inside the embedded event loop.""" if not coroutines.iscoroutine(coro): raise ValueError("a coroutine was expected, got {!r}".format(coro)) return self._loop.run_until_complete(coro) def get_loop(self): + """Returnb embedded event loop.""" return self._loop - def new_loop(self): - return events.new_event_loop() - def run(main, *, debug=None): @@ -70,7 +88,12 @@ async def main(): asyncio.run(main()) """ - with Runner(debug) as runner: + if events._get_running_loop() is not None: + # fail fast with short traceback + raise RuntimeError( + "asyncio.run() cannot be called from a running event loop") + + with Runner(debug=debug) as runner: return runner.run(main) From d5304813730a257e9dfd9051f435704b8b0982b8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 18:20:36 +0200 Subject: [PATCH 04/27] Add tests --- Lib/asyncio/runners.py | 2 + Lib/test/test_asyncio/test_runners.py | 65 +++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 0b39688a3d918c..5e4f9c572e0784 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -43,6 +43,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): def close(self): """Shutdown and close event loop.""" + if self._loop is None: + return try: _cancel_all_tasks(self._loop) self._loop.run_until_complete(self._loop.shutdown_asyncgens()) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 112273662b20b5..45b81db18432f6 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -186,5 +186,70 @@ async def main(): self.assertFalse(spinner.ag_running) +class RunnerTests(BaseTest): + + def test_non_debug(self): + with asyncio.Runner(debug=False) as runner: + self.assertFalse(runner.get_loop().get_debug()) + + def test_debug(self): + with asyncio.Runner(debug=True) as runner: + self.assertTrue(runner.get_loop().get_debug()) + + def test_custom_factory(self): + loop = mock.Mock() + with asyncio.Runner(factory=lambda: loop) as runner: + self.assertIs(runner.get_loop(), loop) + + def test_run(self): + async def f(): + await asyncio.sleep(0) + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(f())) + loop = runner.get_loop() + + self.assertIsNone(runner.get_loop()) + self.assertTrue(loop.is_closed()) + + def test_run_non_coro(self): + with asyncio.Runner() as runner: + with self.assertRaisesRegex( + ValueError, + "a coroutine was expected" + ): + runner.run(123) + + def test_run_future(self): + with asyncio.Runner() as runner: + with self.assertRaisesRegex( + ValueError, + "a coroutine was expected" + ): + fut = runner.get_loop().create_future() + runner.run(fut) + + def test_explicit_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + runner.close() + + self.assertIsNone(runner.get_loop()) + self.assertTrue(loop.is_closed()) + + def test_double_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + + runner.close() + self.assertIsNone(runner.get_loop()) + self.assertTrue(loop.is_closed()) + + runner.close() + self.assertIsNone(runner.get_loop()) + self.assertTrue(loop.is_closed()) + + if __name__ == '__main__': unittest.main() From 6ee7c9a2729804b82a468e81c8108950bdeb3d25 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 18:29:13 +0200 Subject: [PATCH 05/27] Add a comment --- Lib/test/test_asyncio/test_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 45b81db18432f6..47cc8b07004996 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -246,8 +246,8 @@ def test_double_close(self): self.assertIsNone(runner.get_loop()) self.assertTrue(loop.is_closed()) + # the second call is no-op runner.close() - self.assertIsNone(runner.get_loop()) self.assertTrue(loop.is_closed()) From e15d70b5607296fef237ddb9ec925d57c5c107bb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 10 Mar 2022 20:42:01 +0200 Subject: [PATCH 06/27] Adopt IsolatedAsyncioTestCase to use Runner --- Lib/unittest/async_case.py | 77 +++++++++++++------------------------- 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index 3c57bb5cda2c03..196e26a38766a3 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -33,7 +33,7 @@ class IsolatedAsyncioTestCase(TestCase): def __init__(self, methodName='runTest'): super().__init__(methodName) - self._asyncioTestLoop = None + self._asyncioRunner = None self._asyncioCallsQueue = None async def asyncSetUp(self): @@ -74,20 +74,22 @@ def _callCleanup(self, function, *args, **kwargs): self._callMaybeAsync(function, *args, **kwargs) def _callAsync(self, func, /, *args, **kwargs): - assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized' + assert self._asyncioRunner is not None, 'asyncio runner is not initialized' ret = func(*args, **kwargs) assert inspect.isawaitable(ret), f'{func!r} returned non-awaitable' - fut = self._asyncioTestLoop.create_future() + loop = self._asyncioRunner.get_loop() + fut = loop.create_future() self._asyncioCallsQueue.put_nowait((fut, ret)) - return self._asyncioTestLoop.run_until_complete(fut) + return loop.run_until_complete(fut) def _callMaybeAsync(self, func, /, *args, **kwargs): - assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized' + assert self._asyncioRunner is not None, 'asyncio runner is not initialized' ret = func(*args, **kwargs) if inspect.isawaitable(ret): - fut = self._asyncioTestLoop.create_future() + loop = self._asyncioRunner.get_loop() + fut = loop.create_future() self._asyncioCallsQueue.put_nowait((fut, ret)) - return self._asyncioTestLoop.run_until_complete(fut) + return loop.run_until_complete(fut) else: return ret @@ -110,62 +112,35 @@ async def _asyncioLoopRunner(self, fut): if not fut.cancelled(): fut.set_exception(ex) - def _setupAsyncioLoop(self): - assert self._asyncioTestLoop is None, 'asyncio test loop already initialized' - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.set_debug(True) - self._asyncioTestLoop = loop + def _setupAsyncioRunner(self): + assert self._asyncioRunner is None, 'asyncio runner is already initialized' + runner = asyncio.Runner(debug=True) + self._asyncioRunner = runner + loop = runner.get_loop() fut = loop.create_future() self._asyncioCallsTask = loop.create_task(self._asyncioLoopRunner(fut)) loop.run_until_complete(fut) - def _tearDownAsyncioLoop(self): - assert self._asyncioTestLoop is not None, 'asyncio test loop is not initialized' - loop = self._asyncioTestLoop - self._asyncioTestLoop = None + def _tearDownAsyncioRunner(self): + assert self._asyncioRunner is not None, 'asyncio runner is not initialized' + runner = self._asyncioRunner + self._asyncioRunner = None self._asyncioCallsQueue.put_nowait(None) - loop.run_until_complete(self._asyncioCallsQueue.join()) - - try: - # cancel all tasks - to_cancel = asyncio.all_tasks(loop) - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - - loop.run_until_complete( - asyncio.gather(*to_cancel, return_exceptions=True)) - - for task in to_cancel: - if task.cancelled(): - continue - if task.exception() is not None: - loop.call_exception_handler({ - 'message': 'unhandled exception during test shutdown', - 'exception': task.exception(), - 'task': task, - }) - # shutdown asyncgens - loop.run_until_complete(loop.shutdown_asyncgens()) - finally: - asyncio.set_event_loop(None) - loop.close() + runner.run(self._asyncioCallsQueue.join()) + runner.close() def run(self, result=None): - self._setupAsyncioLoop() + self._setupAsyncioRunner() try: return super().run(result) finally: - self._tearDownAsyncioLoop() + self._tearDownAsyncioRunner() def debug(self): - self._setupAsyncioLoop() + self._setupAsyncioRunner() super().debug() - self._tearDownAsyncioLoop() + self._tearDownAsyncioRunner() def __del__(self): - if self._asyncioTestLoop is not None: - self._tearDownAsyncioLoop() + if self._asyncioRunner is not None: + self._tearDownAsyncioRunner() From 8014227bc7437c0e8fe1876f015c175c600392cf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 12 Mar 2022 02:13:17 +0200 Subject: [PATCH 07/27] Add docs sketch --- Doc/library/asyncio-runner.rst | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Doc/library/asyncio-runner.rst diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst new file mode 100644 index 00000000000000..67b4b0f05d2552 --- /dev/null +++ b/Doc/library/asyncio-runner.rst @@ -0,0 +1,49 @@ +.. currentmodule:: asyncio + + +====== +Runner +====== + +**Source code:** :source:`Lib/asyncio/runners.py` + +------------------------------------ + +*Runner* context manager is used for two purposes: + +1. Providing a primitive for event loop initialization and finalization with correct + resources cleanup (cancelling background tasks, shutdowning the default thread-pool + executor and pending async generators, etc.) + +2. Processing *context variables* between async function calls. + +:func:`asyncio.run` is used for running asyncio code usually, but sometimes several +top-level async calls are needed in the same loop and context instead of the +single ``main()`` call provided by :func:`asyncio.run`. + +For example, there is a synchronous unittest library or console framework that should +work with async code. + +A code that The following examples are equal: + +.. code:: python + + async def main(): + ... + + asyncio.run(main()) + + +Usually, + +.. rubric:: Preface + +The event loop is the core of every asyncio application. +Event loops run asynchronous tasks and callbacks, perform network +IO operations, and run subprocesses. + +Application developers should typically use the high-level asyncio functions, +such as :func:`asyncio.run`, and should rarely need to reference the loop +object or call its methods. This section is intended mostly for authors +of lower-level code, libraries, and frameworks, who need finer control over +the event loop behavior. From 98e7a22febe137f2f090e960d0863b47ed67f00e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 14 Mar 2022 15:14:56 +0200 Subject: [PATCH 08/27] Add context arg to Runner.run() --- Lib/asyncio/runners.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 5e4f9c572e0784..f3134c1c966844 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -46,26 +46,27 @@ def close(self): if self._loop is None: return try: - _cancel_all_tasks(self._loop) - self._loop.run_until_complete(self._loop.shutdown_asyncgens()) - self._loop.run_until_complete(self._loop.shutdown_default_executor()) + loop = self._loop + _cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) finally: - self._loop.close() + loop.close() self._loop = None - def run(self, coro): + def run(self, coro, *, context=None): """Run a coroutine inside the embedded event loop.""" if not coroutines.iscoroutine(coro): raise ValueError("a coroutine was expected, got {!r}".format(coro)) - return self._loop.run_until_complete(coro) + task = self._loop.create_task(coro, context=context) + return self._loop.run_until_complete(task) def get_loop(self): """Returnb embedded event loop.""" return self._loop - def run(main, *, debug=None): """Execute the coroutine and return the result. From f4cd673a2209a65cc760baeb686f38126951565b Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 00:39:12 +0200 Subject: [PATCH 09/27] Work on docs --- Doc/library/asyncio-custom-loop.rst | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Doc/library/asyncio-custom-loop.rst diff --git a/Doc/library/asyncio-custom-loop.rst b/Doc/library/asyncio-custom-loop.rst new file mode 100644 index 00000000000000..78f0f9ed20c3bf --- /dev/null +++ b/Doc/library/asyncio-custom-loop.rst @@ -0,0 +1,43 @@ +.. currentmodule:: asyncio + + +.. _asyncio-custom-loop: + +================= +Custom Event Loop +================= + +Asyncio can be extended by a custom event loop (and event loop policy) implemented by +third-party libraries. + + +.. note:: + That third-parties should reuse existing asyncio code + (e.g. ``asyncio.BaseEventLoop``) with caution, + a new Python version can make a change that breaks the backward + compatibility accidentally. + + +Future and Task private constructors +==================================== + +:class:`asyncio.Future` and :class:`asyncio.Task` should be never created directly, +plase use corresponding :meth:`loop.create_future` and :meth:`loop.create_task`, +or `asyncio.create_task` factories instead. + +However, during a customloop implementation the third-party library may *reuse* defaul +highly optimized asyncio future and task implementation. For this purpose, *private* +constructor signatures are listed: + +* ``Future.__init__(*, loop=None)``, where *loop* is an optional event loop instance. + + +* ``Task.__init__(coro, *, loop=None, name=None, context=None)``, where *loop* is an + optional event loop instance. The rest of arguments are described in + :meth:`loop.create_task` description. + + +Task lifetime support +===================== + +I From a443b3799bcb820ad2018655f69ecc8ac4525a3d Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 21:47:07 +0200 Subject: [PATCH 10/27] Work on --- Doc/library/asyncio-runner.rst | 38 ++++++++++++++++++++-------------- Lib/asyncio/runners.py | 17 +++++++++++++-- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 67b4b0f05d2552..244a9084dd34c4 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -9,31 +9,39 @@ Runner ------------------------------------ -*Runner* context manager is used for two purposes: -1. Providing a primitive for event loop initialization and finalization with correct - resources cleanup (cancelling background tasks, shutdowning the default thread-pool - executor and pending async generators, etc.) +:func:`asyncio.run` provides a convinient very high-level API for running asyncio code. -2. Processing *context variables* between async function calls. +It is the preferred approach that satisfies almost all use cases. -:func:`asyncio.run` is used for running asyncio code usually, but sometimes several -top-level async calls are needed in the same loop and context instead of the -single ``main()`` call provided by :func:`asyncio.run`. +Sometimes several top-level async calls are needed in the same loop and contextvars +context instead of the single ``main()`` call provided by :func:`asyncio.run`. -For example, there is a synchronous unittest library or console framework that should -work with async code. - -A code that The following examples are equal: +The *Runner* context manager can be used for such things: .. code:: python - async def main(): - ... + with asyncio.Runner() as runner: + runner.run(func1()) + runner.run(func2()) + +On the :class:`~asyncio.Runner` instantiation the new event loop is created. + +All :meth:`~asyncio.Runner.run` calls share the same :class:`~contextvars.Context` and +internal :class:`~asyncio.loop`. + +On the exit of :keyword:`with` block all background tasks are cancelled, the embedded +loop is closing. + + +.. class:: Runner(*, debug=None, factory=None) + + + - asyncio.run(main()) +enter Usually, .. rubric:: Preface diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index f3134c1c966844..28baacb0bb4787 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -1,5 +1,6 @@ __all__ = ('Runner', 'run') +import contextvars from . import coroutines from . import events from . import tasks @@ -30,10 +31,14 @@ class Runner: asyncio.run() call doesn't work. """ - def __init__(self, *, debug=None, factory=events.new_event_loop): - self._loop = factory() + def __init__(self, *, debug=None, factory=None): + if factory is None: + self._loop = events.new_event_loop() + else: + self._loop = factory() if debug is not None: self._loop.set_debug(debug) + self._context = contextvars.copy_context() def __enter__(self): return self @@ -59,6 +64,11 @@ def run(self, coro, *, context=None): if not coroutines.iscoroutine(coro): raise ValueError("a coroutine was expected, got {!r}".format(coro)) + if self._loop is None: + raise RuntimeError("Runner is closed") + + if context is None: + context = self._context task = self._loop.create_task(coro, context=context) return self._loop.run_until_complete(task) @@ -66,6 +76,9 @@ def get_loop(self): """Returnb embedded event loop.""" return self._loop + def get_context(self): + return self._context.copy() + def run(main, *, debug=None): """Execute the coroutine and return the result. From e6be8f72e4c8be69b72999c4cbb375efca63b029 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 22:37:24 +0200 Subject: [PATCH 11/27] Improve docs --- Doc/library/asyncio-runner.rst | 110 ++++++++++++++++++++++++--------- Doc/library/asyncio-task.rst | 37 ----------- Doc/library/asyncio.rst | 1 + 3 files changed, 82 insertions(+), 66 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 244a9084dd34c4..0fe649cf102f97 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -1,57 +1,109 @@ .. currentmodule:: asyncio -====== -Runner -====== +======= +Runners +======= **Source code:** :source:`Lib/asyncio/runners.py` ------------------------------------- +This section outlines high-level asyncio primitives to run asyncio code. -:func:`asyncio.run` provides a convinient very high-level API for running asyncio code. +They are built on top of :ref:`event loop ` with the aim to simplify +async code usage for common wide-spread scenarion. -It is the preferred approach that satisfies almost all use cases. +.. contents:: + :depth: 1 + :local: -Sometimes several top-level async calls are needed in the same loop and contextvars -context instead of the single ``main()`` call provided by :func:`asyncio.run`. -The *Runner* context manager can be used for such things: -.. code:: python +Running an asyncio Program +========================== - with asyncio.Runner() as runner: - runner.run(func1()) - runner.run(func2()) +.. function:: run(coro, *, debug=None) -On the :class:`~asyncio.Runner` instantiation the new event loop is created. + Execute the :term:`coroutine` *coro* and return the result. -All :meth:`~asyncio.Runner.run` calls share the same :class:`~contextvars.Context` and -internal :class:`~asyncio.loop`. + This function runs the passed coroutine, taking care of + managing the asyncio event loop, *finalizing asynchronous + generators*, and closing the threadpool. -On the exit of :keyword:`with` block all background tasks are cancelled, the embedded -loop is closing. + This function cannot be called when another asyncio event loop is + running in the same thread. + If *debug* is ``True``, the event loop will be run in debug mode. ``False`` disables + debug mode explicitly. ``None`` is used to respect the global + :ref:`asyncio-debug-mode` settings. + + This function always creates a new event loop and closes it at + the end. It should be used as a main entry point for asyncio + programs, and should ideally only be called once. + + Example:: + + async def main(): + await asyncio.sleep(1) + print('hello') + + asyncio.run(main()) + + .. versionadded:: 3.7 + + .. versionchanged:: 3.9 + Updated to use :meth:`loop.shutdown_default_executor`. + + .. versionchanged:: 3.10 + + *debug* is ``None`` by default to respect the global debug mode settings. + + +Runner context manager +====================== .. class:: Runner(*, debug=None, factory=None) + A context manager that simplifies *multiple* async function calls in the same + context. + + Sometimes several top-level async functions should be called in the same :ref:`event + loop ` and :class:`contextvars.Context`. + + If *debug* is ``True``, the event loop will be run in debug mode. ``False`` disables + debug mode explicitly. ``None`` is used to respect the global + :ref:`asyncio-debug-mode` settings. + + *factory* could be used for overriding the loop creation. + :func:`asyncio.new_event_loop` is used if ``None``. + + Basically, :func:`asyncio.run()` example can be revealed with the runner usage: + + .. block:: python + + async def main(): + await asyncio.sleep(1) + print('hello') + + with asyncio.Runner() as runner: + runner.run(main()) + + .. versionadded:: 3.11 + .. method:: run(coro, *, context=None) + Run a :term:`coroutine ` *coro* in the embedded loop. + Return the coroutine's result or raise its exception. + An optional keyword-only *context* argument allows specifying a + custom :class:`contextvars.Context` for the *coro* to run in. + The runner's context is used if ``None``. -enter -Usually, + .. method:: get_loop() -.. rubric:: Preface + Return the event loop associated with the runner instance. -The event loop is the core of every asyncio application. -Event loops run asynchronous tasks and callbacks, perform network -IO operations, and run subprocesses. + .. method:: get_context() -Application developers should typically use the high-level asyncio functions, -such as :func:`asyncio.run`, and should rarely need to reference the loop -object or call its methods. This section is intended mostly for authors -of lower-level code, libraries, and frameworks, who need finer control over -the event loop behavior. + Return the :class:`contextvars.Context` associated with the runner object. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 294f5ab2b22f9d..6d0ceedb72e4a3 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -204,43 +204,6 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Running an asyncio Program -========================== - -.. function:: run(coro, *, debug=False) - - Execute the :term:`coroutine` *coro* and return the result. - - This function runs the passed coroutine, taking care of - managing the asyncio event loop, *finalizing asynchronous - generators*, and closing the threadpool. - - This function cannot be called when another asyncio event loop is - running in the same thread. - - If *debug* is ``True``, the event loop will be run in debug mode. - - This function always creates a new event loop and closes it at - the end. It should be used as a main entry point for asyncio - programs, and should ideally only be called once. - - Example:: - - async def main(): - await asyncio.sleep(1) - print('hello') - - asyncio.run(main()) - - .. versionadded:: 3.7 - - .. versionchanged:: 3.9 - Updated to use :meth:`loop.shutdown_default_executor`. - - .. note:: - The source code for ``asyncio.run()`` can be found in - :source:`Lib/asyncio/runners.py`. - Creating Tasks ============== diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 94a853259d3483..8b3a060ffad526 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -67,6 +67,7 @@ Additionally, there are **low-level** APIs for :caption: High-level APIs :maxdepth: 1 + asyncio-runner.rst asyncio-task.rst asyncio-stream.rst asyncio-sync.rst From b1dfe4f1370c426bc03134af35beaa3ccc46e978 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 22:46:26 +0200 Subject: [PATCH 12/27] Add NEWS --- .../NEWS.d/next/Library/2022-03-18-22-46-18.bpo-47062.RNc99_.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2022-03-18-22-46-18.bpo-47062.RNc99_.rst diff --git a/Misc/NEWS.d/next/Library/2022-03-18-22-46-18.bpo-47062.RNc99_.rst b/Misc/NEWS.d/next/Library/2022-03-18-22-46-18.bpo-47062.RNc99_.rst new file mode 100644 index 00000000000000..7d5bfc114a8d17 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-03-18-22-46-18.bpo-47062.RNc99_.rst @@ -0,0 +1 @@ +Implement :class:`asyncio.Runner` context manager. From 759f72ade7407772cff931cca19b61d09317f75e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 22:55:45 +0200 Subject: [PATCH 13/27] Drop not related file : --- Doc/library/asyncio-custom-loop.rst | 43 ----------------------------- 1 file changed, 43 deletions(-) delete mode 100644 Doc/library/asyncio-custom-loop.rst diff --git a/Doc/library/asyncio-custom-loop.rst b/Doc/library/asyncio-custom-loop.rst deleted file mode 100644 index 78f0f9ed20c3bf..00000000000000 --- a/Doc/library/asyncio-custom-loop.rst +++ /dev/null @@ -1,43 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-custom-loop: - -================= -Custom Event Loop -================= - -Asyncio can be extended by a custom event loop (and event loop policy) implemented by -third-party libraries. - - -.. note:: - That third-parties should reuse existing asyncio code - (e.g. ``asyncio.BaseEventLoop``) with caution, - a new Python version can make a change that breaks the backward - compatibility accidentally. - - -Future and Task private constructors -==================================== - -:class:`asyncio.Future` and :class:`asyncio.Task` should be never created directly, -plase use corresponding :meth:`loop.create_future` and :meth:`loop.create_task`, -or `asyncio.create_task` factories instead. - -However, during a customloop implementation the third-party library may *reuse* defaul -highly optimized asyncio future and task implementation. For this purpose, *private* -constructor signatures are listed: - -* ``Future.__init__(*, loop=None)``, where *loop* is an optional event loop instance. - - -* ``Task.__init__(coro, *, loop=None, name=None, context=None)``, where *loop* is an - optional event loop instance. The rest of arguments are described in - :meth:`loop.create_task` description. - - -Task lifetime support -===================== - -I From f47d66ac9041d3123f140ae125b4b457536eab20 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 18 Mar 2022 23:22:17 +0200 Subject: [PATCH 14/27] Fix doc --- Doc/library/asyncio-runner.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 0fe649cf102f97..fda2982ce1a1c9 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -77,9 +77,7 @@ Runner context manager *factory* could be used for overriding the loop creation. :func:`asyncio.new_event_loop` is used if ``None``. - Basically, :func:`asyncio.run()` example can be revealed with the runner usage: - - .. block:: python + Basically, :func:`asyncio.run()` example can be revealed with the runner usage:: async def main(): await asyncio.sleep(1) From 546440b2c591f10999a121c60025e017df460115 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 00:11:50 +0200 Subject: [PATCH 15/27] Update Lib/asyncio/runners.py Co-authored-by: Zachary Ware --- Lib/asyncio/runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 28baacb0bb4787..dde663b84a7c7c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -73,7 +73,7 @@ def run(self, coro, *, context=None): return self._loop.run_until_complete(task) def get_loop(self): - """Returnb embedded event loop.""" + """Return embedded event loop.""" return self._loop def get_context(self): From 6935f7dc9f05a18a95e320c63d2f39d7448c89ce Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 00:12:33 +0200 Subject: [PATCH 16/27] Update Doc/library/asyncio-runner.rst Co-authored-by: Zachary Ware --- Doc/library/asyncio-runner.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index fda2982ce1a1c9..b4a8a0f2a72716 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -10,8 +10,8 @@ Runners This section outlines high-level asyncio primitives to run asyncio code. -They are built on top of :ref:`event loop ` with the aim to simplify -async code usage for common wide-spread scenarion. +They are built on top of an :ref:`event loop ` with the aim +to simplify async code usage for common wide-spread scenarios. .. contents:: :depth: 1 From 9b9a0043faf03efadecb1a0a8dcaccae6252ca8a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 00:12:38 +0200 Subject: [PATCH 17/27] Update Lib/asyncio/runners.py Co-authored-by: Zachary Ware --- Lib/asyncio/runners.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index dde663b84a7c7c..262e02d9e916ed 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -22,8 +22,7 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - - .run() method can be called multiple times. + The run() method can be called multiple times within the runner's context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code From b0c5b8c3ec32b1ed87cdb63f1bdefe024f8d63f1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 14:09:44 +0200 Subject: [PATCH 18/27] Improve wording --- Doc/library/asyncio-runner.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index b4a8a0f2a72716..66b314f57cbb14 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -77,7 +77,7 @@ Runner context manager *factory* could be used for overriding the loop creation. :func:`asyncio.new_event_loop` is used if ``None``. - Basically, :func:`asyncio.run()` example can be revealed with the runner usage:: + Basically, :func:`asyncio.run()` example can be rewritten with the runner usage:: async def main(): await asyncio.sleep(1) From 599c9dbdbefb6f9a62649478d37b080ccb74a313 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 14:15:46 +0200 Subject: [PATCH 19/27] Add a test for double 'with' usage --- Lib/asyncio/runners.py | 5 +++-- Lib/test/test_asyncio/test_runners.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 262e02d9e916ed..9fdc62d3395857 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -9,8 +9,9 @@ class Runner: """A context manager that controls event loop life cycle. - The context manager always creates a new event loop, allows to run async funtions - inside it, and properly finalizes the loop at the context manager exit. + The context manager always creates a new event loop, + allows to run async functions inside it, + and properly finalizes the loop at the context manager exit. If debug is True, the event loop will be run in debug mode. If factory is passed, it is used for new event loop creation. diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 47cc8b07004996..d51cf0d3165edd 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -250,6 +250,25 @@ def test_double_close(self): runner.close() self.assertTrue(loop.is_closed()) + def test_second_with_block_raises(self): + ret = [] + + async def f(arg): + ret.append(arg) + + runner = asyncio.Runner() + with runner: + runner.run(f(1)) + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + with runner: + runner.run(f(2)) + + self.assertEqual([1], ret) + if __name__ == '__main__': unittest.main() From b0da74b648016f7f9e17ac5e8ba986ca0e08fd9a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 19 Mar 2022 14:47:09 +0200 Subject: [PATCH 20/27] Improve tests --- Lib/test/test_asyncio/test_runners.py | 28 +++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index d51cf0d3165edd..049028c2859d36 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import unittest from unittest import mock @@ -260,15 +261,34 @@ async def f(arg): with runner: runner.run(f(1)) - with self.assertRaisesRegex( + with self.assertWarnsRegex( + RuntimeWarning, + "coroutine .+ was never awaited" # f(2) is not executed + ): + with self.assertRaisesRegex( RuntimeError, "Runner is closed" - ): - with runner: - runner.run(f(2)) + ): + with runner: + runner.run(f(2)) self.assertEqual([1], ret) + def test_run_keeps_context(self): + cvar = contextvars.ContextVar("cvar", default=-1) + + async def f(val): + old = cvar.get() + await asyncio.sleep(0) + cvar.set(val) + return old + + with asyncio.Runner() as runner: + self.assertEqual(-1, runner.run(f(1))) + self.assertEqual(1, runner.run(f(2))) + + self.assertEqual({cvar: 2}, dict(runner.get_context().items())) + if __name__ == '__main__': unittest.main() From 04cfff937dc690aba0dd2d64ce4e3b3a3f90e2cb Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 22 Mar 2022 17:58:21 +0200 Subject: [PATCH 21/27] Work on --- Lib/asyncio/runners.py | 48 +++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 9fdc62d3395857..55590780e37fda 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -1,11 +1,18 @@ __all__ = ('Runner', 'run') import contextvars +import enum from . import coroutines from . import events from . import tasks +class _State(enum.Enum): + CREATED = "created" + INITIALIZED = "initialized" + CLOSED = "closed" + + class Runner: """A context manager that controls event loop life cycle. @@ -32,15 +39,14 @@ class Runner: """ def __init__(self, *, debug=None, factory=None): - if factory is None: - self._loop = events.new_event_loop() - else: - self._loop = factory() - if debug is not None: - self._loop.set_debug(debug) - self._context = contextvars.copy_context() + self._state = _State.CREATED + self._debug = debug + self._factory = factory + self._loop = None + self._context = None def __enter__(self): + self._lazy_init() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -48,7 +54,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def close(self): """Shutdown and close event loop.""" - if self._loop is None: + if self._state is not _state.INITIALIZED: return try: loop = self._loop @@ -58,14 +64,14 @@ def close(self): finally: loop.close() self._loop = None + self._context = None def run(self, coro, *, context=None): """Run a coroutine inside the embedded event loop.""" if not coroutines.iscoroutine(coro): raise ValueError("a coroutine was expected, got {!r}".format(coro)) - if self._loop is None: - raise RuntimeError("Runner is closed") + self._lazy_init() if context is None: context = self._context @@ -74,11 +80,33 @@ def run(self, coro, *, context=None): def get_loop(self): """Return embedded event loop.""" + self._check() return self._loop def get_context(self): + self._check() return self._context.copy() + def _check(self): + if self._state is _State.CREATED: + raise RuntimeError("Runner is not initialized") + if self._state is _State.CLOSED: + raise RuntimeError("Runner is closed") + + def _lazy_init(self): + if self._state is _State.CLOSED: + raise RuntimeError("Runner is closed") + if self._state is _State.INITIALIZED: + return + if self._factory is None: + self._loop = events.new_event_loop() + else: + self._loop = self._factory() + if self._debug is not None: + self._loop.set_debug(self._debug) + self._context = contextvars.copy_context() + + def run(main, *, debug=None): """Execute the coroutine and return the result. From 674ad4e0e21f72e2b1395c8337baa4061bcdb557 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 22 Mar 2022 18:37:41 +0200 Subject: [PATCH 22/27] Lazy init version --- Lib/asyncio/runners.py | 15 ++++------ Lib/test/test_asyncio/test_runners.py | 42 ++++++++++++++++++--------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 55590780e37fda..3bc83da54bf9d1 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -54,7 +54,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def close(self): """Shutdown and close event loop.""" - if self._state is not _state.INITIALIZED: + if self._state is not _State.INITIALIZED: return try: loop = self._loop @@ -64,7 +64,7 @@ def close(self): finally: loop.close() self._loop = None - self._context = None + self._state = _State.CLOSED def run(self, coro, *, context=None): """Run a coroutine inside the embedded event loop.""" @@ -80,19 +80,13 @@ def run(self, coro, *, context=None): def get_loop(self): """Return embedded event loop.""" - self._check() + self._lazy_init() return self._loop def get_context(self): - self._check() + self._lazy_init() return self._context.copy() - def _check(self): - if self._state is _State.CREATED: - raise RuntimeError("Runner is not initialized") - if self._state is _State.CLOSED: - raise RuntimeError("Runner is closed") - def _lazy_init(self): if self._state is _State.CLOSED: raise RuntimeError("Runner is closed") @@ -105,6 +99,7 @@ def _lazy_init(self): if self._debug is not None: self._loop.set_debug(self._debug) self._context = contextvars.copy_context() + self._state = _State.INITIALIZED diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 049028c2859d36..a2f8759ea08839 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -211,7 +211,18 @@ async def f(): self.assertEqual('done', runner.run(f())) loop = runner.get_loop() - self.assertIsNone(runner.get_loop()) + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_context() + self.assertTrue(loop.is_closed()) def test_run_non_coro(self): @@ -235,8 +246,18 @@ def test_explicit_close(self): runner = asyncio.Runner() loop = runner.get_loop() runner.close() + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_context() - self.assertIsNone(runner.get_loop()) self.assertTrue(loop.is_closed()) def test_double_close(self): @@ -244,7 +265,6 @@ def test_double_close(self): loop = runner.get_loop() runner.close() - self.assertIsNone(runner.get_loop()) self.assertTrue(loop.is_closed()) # the second call is no-op @@ -261,16 +281,12 @@ async def f(arg): with runner: runner.run(f(1)) - with self.assertWarnsRegex( - RuntimeWarning, - "coroutine .+ was never awaited" # f(2) is not executed + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" ): - with self.assertRaisesRegex( - RuntimeError, - "Runner is closed" - ): - with runner: - runner.run(f(2)) + with runner: + runner.run(f(2)) self.assertEqual([1], ret) @@ -287,7 +303,7 @@ async def f(val): self.assertEqual(-1, runner.run(f(1))) self.assertEqual(1, runner.run(f(2))) - self.assertEqual({cvar: 2}, dict(runner.get_context().items())) + self.assertEqual({cvar: 2}, dict(runner.get_context().items())) if __name__ == '__main__': From 7cd5430dccd89da0c2034f2cd9506ea2ff9cd610 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 22 Mar 2022 20:57:29 +0200 Subject: [PATCH 23/27] Tune --- Doc/library/asyncio-runner.rst | 14 ++++++++++++-- Lib/asyncio/runners.py | 19 ++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 66b314f57cbb14..c834a284984f96 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -96,7 +96,9 @@ Runner context manager An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *coro* to run in. - The runner's context is used if ``None``. + The runner's default context is used if ``None``. + + .. method:: close() .. method:: get_loop() @@ -104,4 +106,12 @@ Runner context manager .. method:: get_context() - Return the :class:`contextvars.Context` associated with the runner object. + Return the default :class:`contextvars.Context` associated with the runner object. + + .. note:: + + :class:`Runner` uses the lazy initialization strategy, its constructor doesn't + initialize underlying low-level structures. + + Embedded *loop* and *context* are created at :keyword:`with` body entering or the + first call of :meth:`run`, :meth:`get_loop`, or :meth:`get_context`. diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 3bc83da54bf9d1..e5195d06bac525 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -66,6 +66,16 @@ def close(self): self._loop = None self._state = _State.CLOSED + def get_context(self): + """Return the default associated context.""" + self._lazy_init() + return self._context.copy() + + def get_loop(self): + """Return embedded event loop.""" + self._lazy_init() + return self._loop + def run(self, coro, *, context=None): """Run a coroutine inside the embedded event loop.""" if not coroutines.iscoroutine(coro): @@ -78,15 +88,6 @@ def run(self, coro, *, context=None): task = self._loop.create_task(coro, context=context) return self._loop.run_until_complete(task) - def get_loop(self): - """Return embedded event loop.""" - self._lazy_init() - return self._loop - - def get_context(self): - self._lazy_init() - return self._context.copy() - def _lazy_init(self): if self._state is _State.CLOSED: raise RuntimeError("Runner is closed") From dd28ef789b5b891a0e444098188829a36ec7a4dd Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 23 Mar 2022 14:45:48 +0200 Subject: [PATCH 24/27] Drop explicit get_context() function, asyncio.Task has no it also --- Doc/library/asyncio-runner.rst | 6 +----- Lib/asyncio/runners.py | 5 ----- Lib/test/test_asyncio/test_runners.py | 17 ++++------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index c834a284984f96..3f9f5bf3ddcb34 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -104,14 +104,10 @@ Runner context manager Return the event loop associated with the runner instance. - .. method:: get_context() - - Return the default :class:`contextvars.Context` associated with the runner object. - .. note:: :class:`Runner` uses the lazy initialization strategy, its constructor doesn't initialize underlying low-level structures. Embedded *loop* and *context* are created at :keyword:`with` body entering or the - first call of :meth:`run`, :meth:`get_loop`, or :meth:`get_context`. + first call of :meth:`run` or :meth:`get_loop`. diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index e5195d06bac525..5572b50fb4e0fe 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -66,11 +66,6 @@ def close(self): self._loop = None self._state = _State.CLOSED - def get_context(self): - """Return the default associated context.""" - self._lazy_init() - return self._context.copy() - def get_loop(self): """Return embedded event loop.""" self._lazy_init() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index a2f8759ea08839..f6156ae72f4ef0 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -217,12 +217,6 @@ async def f(): ): runner.get_loop() - with self.assertRaisesRegex( - RuntimeError, - "Runner is closed" - ): - runner.get_context() - self.assertTrue(loop.is_closed()) def test_run_non_coro(self): @@ -252,12 +246,6 @@ def test_explicit_close(self): ): runner.get_loop() - with self.assertRaisesRegex( - RuntimeError, - "Runner is closed" - ): - runner.get_context() - self.assertTrue(loop.is_closed()) def test_double_close(self): @@ -299,11 +287,14 @@ async def f(val): cvar.set(val) return old + async def get_context(): + return contextvars.copy_context() + with asyncio.Runner() as runner: self.assertEqual(-1, runner.run(f(1))) self.assertEqual(1, runner.run(f(2))) - self.assertEqual({cvar: 2}, dict(runner.get_context().items())) + self.assertEqual({cvar: 2}, dict(runner.run(get_context()))) if __name__ == '__main__': From 5e13b2ea5c39869d1b959c5d823bb805b1454bc5 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 23 Mar 2022 16:22:15 +0200 Subject: [PATCH 25/27] Add docs for .close() method --- Doc/library/asyncio-runner.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 3f9f5bf3ddcb34..2f4de9edaa400d 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -98,8 +98,16 @@ Runner context manager custom :class:`contextvars.Context` for the *coro* to run in. The runner's default context is used if ``None``. + This function cannot be called when another asyncio event loop is + running in the same thread. + .. method:: close() + Close the runner. + + Finalize asynchronous generators, shutdown default executor, close the event loop + and release embedded :class:`contextvars.Context`. + .. method:: get_loop() Return the event loop associated with the runner instance. @@ -109,5 +117,5 @@ Runner context manager :class:`Runner` uses the lazy initialization strategy, its constructor doesn't initialize underlying low-level structures. - Embedded *loop* and *context* are created at :keyword:`with` body entering or the - first call of :meth:`run` or :meth:`get_loop`. + Embedded *loop* and *context* are created at the :keyword:`with` body entering + or the first call of :meth:`run` or :meth:`get_loop`. From c0b999d344ac4593917575202d0be1ccc665c4b4 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 24 Mar 2022 17:09:49 +0200 Subject: [PATCH 26/27] Add better error message for recursive run() call --- Lib/asyncio/runners.py | 5 +++++ Lib/test/test_asyncio/test_runners.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 5572b50fb4e0fe..2f1becd3ef0c2d 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -76,6 +76,11 @@ def run(self, coro, *, context=None): if not coroutines.iscoroutine(coro): raise ValueError("a coroutine was expected, got {!r}".format(coro)) + if events._get_running_loop() is not None: + # fail fast with short traceback + raise RuntimeError( + "Runner.run() cannot be called from a running event loop") + self._lazy_init() if context is None: diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index f6156ae72f4ef0..2919412ab81db1 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -1,5 +1,7 @@ import asyncio import contextvars +import gc +import re import unittest from unittest import mock @@ -296,6 +298,26 @@ async def get_context(): self.assertEqual({cvar: 2}, dict(runner.run(get_context()))) + def test_recursine_run(self): + async def g(): + pass + + async def f(): + runner.run(g()) + + with asyncio.Runner() as runner: + with self.assertWarnsRegex( + RuntimeWarning, + "coroutine .+ was never awaited", + ): + with self.assertRaisesRegex( + RuntimeError, + re.escape( + "Runner.run() cannot be called from a running event loop" + ), + ): + runner.run(f()) + if __name__ == '__main__': unittest.main() From 4937cd0d292d2cb1c6a11995658d6fdaa1dc5a65 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 24 Mar 2022 17:20:35 +0200 Subject: [PATCH 27/27] Add a note --- Lib/asyncio/runners.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 2f1becd3ef0c2d..975509c7d645d5 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -38,6 +38,9 @@ class Runner: asyncio.run() call doesn't work. """ + + # Note: the class is final, it is not intended for inheritance. + def __init__(self, *, debug=None, factory=None): self._state = _State.CREATED self._debug = debug