-
-
Notifications
You must be signed in to change notification settings - Fork 373
[v3] Sync with futures #1804
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[v3] Sync with futures #1804
Changes from 1 commit
3ad2be3
07bba03
2411640
b8358c3
6470232
c2a0c1d
4fe9456
43402dc
67dd040
d3b81cf
5679b07
778659f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,27 +1,29 @@ | ||||||
| from __future__ import annotations | ||||||
|
|
||||||
| import asyncio | ||||||
| from concurrent.futures import wait | ||||||
| import threading | ||||||
| from typing import ( | ||||||
| Any, | ||||||
| AsyncIterator, | ||||||
| Coroutine, | ||||||
| List, | ||||||
| Optional, | ||||||
| TypeVar, | ||||||
| ) | ||||||
| from typing_extensions import ParamSpec | ||||||
|
|
||||||
| from zarr.v3.config import SyncConfiguration | ||||||
|
|
||||||
| P = ParamSpec("P") | ||||||
| T = TypeVar("T") | ||||||
|
|
||||||
| # From https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py | ||||||
|
|
||||||
| iothread: List[Optional[threading.Thread]] = [None] # dedicated IO thread | ||||||
| loop: List[Optional[asyncio.AbstractEventLoop]] = [ | ||||||
| iothread: list[threading.Thread | None] = [None] # dedicated IO thread | ||||||
| loop: list[asyncio.AbstractEventLoop | None] = [ | ||||||
| None | ||||||
| ] # global event loop for any non-async instance | ||||||
| _lock: Optional[threading.Lock] = None # global lock placeholder | ||||||
| _lock: threading.Lock | None = None # global lock placeholder | ||||||
| get_running_loop = asyncio.get_running_loop | ||||||
|
|
||||||
|
|
||||||
|
|
@@ -36,16 +38,18 @@ def _get_lock() -> threading.Lock: | |||||
| return _lock | ||||||
|
|
||||||
|
|
||||||
| async def _runner(event: threading.Event, coro: Coroutine, result_box: List[Optional[Any]]): | ||||||
| async def _runner(coro: Coroutine[Any, Any, T]) -> T | BaseException: | ||||||
| """ | ||||||
| Await a coroutine and return the result of running it. If await it raises an exception, | ||||||
| that will be returned instead. | ||||||
| """ | ||||||
| try: | ||||||
| result_box[0] = await coro | ||||||
| return await coro | ||||||
| except Exception as ex: | ||||||
| result_box[0] = ex | ||||||
| finally: | ||||||
| event.set() | ||||||
| return ex | ||||||
|
|
||||||
|
|
||||||
| def sync(coro: Coroutine, loop: Optional[asyncio.AbstractEventLoop] = None): | ||||||
| def sync(coro: Coroutine[Any, Any, T], loop: asyncio.AbstractEventLoop | None = None) -> T: | ||||||
|
||||||
| """ | ||||||
| Make loop run coroutine until it returns. Runs in other thread | ||||||
|
|
||||||
|
|
@@ -65,22 +69,19 @@ def sync(coro: Coroutine, loop: Optional[asyncio.AbstractEventLoop] = None): | |||||
| raise NotImplementedError("Calling sync() from within a running loop") | ||||||
| except RuntimeError: | ||||||
| pass | ||||||
| result_box: List[Optional[Any]] = [None] | ||||||
| event = threading.Event() | ||||||
| asyncio.run_coroutine_threadsafe(_runner(event, coro, result_box), loop) | ||||||
| while True: | ||||||
| # this loops allows thread to get interrupted | ||||||
|
||||||
| if event.wait(1): | ||||||
| break | ||||||
|
|
||||||
| return_result = result_box[0] | ||||||
|
|
||||||
| future = asyncio.run_coroutine_threadsafe(_runner(coro), loop) | ||||||
|
|
||||||
| done, _ = wait([future]) | ||||||
| return_result = list(done)[0].result() | ||||||
|
||||||
|
|
||||||
| if isinstance(return_result, BaseException): | ||||||
| raise return_result | ||||||
| else: | ||||||
| return return_result | ||||||
|
|
||||||
|
|
||||||
| def _get_loop(): | ||||||
| def _get_loop() -> asyncio.AbstractEventLoop | None: | ||||||
|
||||||
| def _get_loop() -> asyncio.AbstractEventLoop | None: | |
| def _get_loop() -> asyncio.AbstractEventLoop: |
I suspect you had this here because of some mypy thing but it should be possible to get this to always return a loop. Perhaps an assert loop[0] is not None right before the return is all that is needed?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Might depend on whether it is called in the main thread, the IO thread of elsewhere,
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So we are targeting py >=3.9?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
3.10+ actually!