From 99cc86c46701c86f9b3a3188fce0d6dcba6c9592 Mon Sep 17 00:00:00 2001 From: Josh Staiger Date: Fri, 3 Nov 2017 12:12:59 -0700 Subject: [PATCH 1/5] Base concurrent.futures.Executor on ContextManager Update the concurrent.futures so that Executor, ThreadPoolExecutor, and ProcessPoolExecutor inherit from the typing.ContextManager[T] generic introduced in Python 3.6. Specifically, this solves a problem in code such as: with ThreadPoolExecutor() as p: ... Where the p variable would be typed as an abstract `Executor`, rather than the specific `ThreadPoolExecutor` as expected. --- stdlib/3/concurrent/futures/_base.pyi | 25 +++++++++++++++++++------ stdlib/3/concurrent/futures/process.pyi | 12 ++++++++++-- stdlib/3/concurrent/futures/thread.pyi | 11 ++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/stdlib/3/concurrent/futures/_base.pyi b/stdlib/3/concurrent/futures/_base.pyi index 836756d5d862..5e6c6f42d44a 100644 --- a/stdlib/3/concurrent/futures/_base.pyi +++ b/stdlib/3/concurrent/futures/_base.pyi @@ -1,5 +1,7 @@ from typing import TypeVar, Generic, Any, Iterable, Iterator, Callable, Tuple, Optional, Set, NamedTuple +import sys + FIRST_COMPLETED = ... # type: str FIRST_EXCEPTION = ... # type: str ALL_COMPLETED = ... # type: str @@ -31,12 +33,23 @@ class Future(Generic[_T]): def set_result(self, result: _T) -> None: ... def set_exception(self, exception: Optional[BaseException]) -> None: ... -class Executor: - def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... - def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... - def shutdown(self, wait: bool = ...) -> None: ... - def __enter__(self) -> Executor: ... - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool: ... + +if sys.version_info >= (3, 6): + from typing import ContextManager + _S = TypeVar('_S') + + class Executor(ContextManager[_S]): + def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... + def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... + def shutdown(self, wait: bool = ...) -> None: ... + +else: + class Executor: + def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... + def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... + def shutdown(self, wait: bool = ...) -> None: ... + def __enter__(self) -> Executor: ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool: ... def as_completed(fs: Iterable[Future[_T]], timeout: Optional[float] = ...) -> Iterator[Future[_T]]: ... diff --git a/stdlib/3/concurrent/futures/process.pyi b/stdlib/3/concurrent/futures/process.pyi index 8076c2d0588e..f43ed703c8f6 100644 --- a/stdlib/3/concurrent/futures/process.pyi +++ b/stdlib/3/concurrent/futures/process.pyi @@ -1,9 +1,17 @@ from typing import Optional, Any + +import sys + from ._base import Future, Executor EXTRA_QUEUED_CALLS = ... # type: Any class BrokenProcessPool(RuntimeError): ... -class ProcessPoolExecutor(Executor): - def __init__(self, max_workers: Optional[int] = ...) -> None: ... +if sys.version_info >= (3,6): + class ProcessPoolExecutor(Executor[ProcessPoolExecutor]): + def __init__(self, max_workers: Optional[int] = ...) -> None: ... + +else: + class ProcessPoolExecutor(Executor): + def __init__(self, max_workers: Optional[int] = ...) -> None: ... diff --git a/stdlib/3/concurrent/futures/thread.pyi b/stdlib/3/concurrent/futures/thread.pyi index 88a45a3e070a..f6dd78f018c6 100644 --- a/stdlib/3/concurrent/futures/thread.pyi +++ b/stdlib/3/concurrent/futures/thread.pyi @@ -2,9 +2,10 @@ from typing import Optional from ._base import Executor, Future import sys -class ThreadPoolExecutor(Executor): - if sys.version_info >= (3, 6): - def __init__(self, max_workers: Optional[int] = ..., - thread_name_prefix: str = ...) -> None: ... - else: +if sys.version_info >= (3, 6): + class ThreadPoolExecutor(Executor[ThreadPoolExecutor]): + def __init__(self, max_workers: Optional[int] = ..., thread_name_prefix: str = ...) -> None: ... + +else: + class ThreadPoolExecutor(Executor): def __init__(self, max_workers: Optional[int] = ...) -> None: ... From 260c1ba23d15360d85d210bb4d916a4cb31975e2 Mon Sep 17 00:00:00 2001 From: Josh Staiger Date: Fri, 3 Nov 2017 13:28:18 -0700 Subject: [PATCH 2/5] Fix flake8 complaint. --- stdlib/3/concurrent/futures/process.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/3/concurrent/futures/process.pyi b/stdlib/3/concurrent/futures/process.pyi index f43ed703c8f6..37223d23535a 100644 --- a/stdlib/3/concurrent/futures/process.pyi +++ b/stdlib/3/concurrent/futures/process.pyi @@ -8,7 +8,7 @@ EXTRA_QUEUED_CALLS = ... # type: Any class BrokenProcessPool(RuntimeError): ... -if sys.version_info >= (3,6): +if sys.version_info >= (3, 6): class ProcessPoolExecutor(Executor[ProcessPoolExecutor]): def __init__(self, max_workers: Optional[int] = ...) -> None: ... From cacaa2c986d32967aa3d297927a37c86bc3ce0a3 Mon Sep 17 00:00:00 2001 From: Josh Staiger Date: Sat, 4 Nov 2017 13:10:52 -0700 Subject: [PATCH 3/5] Revert "Fix flake8 complaint." This reverts commit 260c1ba23d15360d85d210bb4d916a4cb31975e2. --- stdlib/3/concurrent/futures/process.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/3/concurrent/futures/process.pyi b/stdlib/3/concurrent/futures/process.pyi index 37223d23535a..f43ed703c8f6 100644 --- a/stdlib/3/concurrent/futures/process.pyi +++ b/stdlib/3/concurrent/futures/process.pyi @@ -8,7 +8,7 @@ EXTRA_QUEUED_CALLS = ... # type: Any class BrokenProcessPool(RuntimeError): ... -if sys.version_info >= (3, 6): +if sys.version_info >= (3,6): class ProcessPoolExecutor(Executor[ProcessPoolExecutor]): def __init__(self, max_workers: Optional[int] = ...) -> None: ... From 89ec503d65d372dd78966b9f813101d2eefe556c Mon Sep 17 00:00:00 2001 From: Josh Staiger Date: Sat, 4 Nov 2017 13:11:34 -0700 Subject: [PATCH 4/5] Revert "Base concurrent.futures.Executor on ContextManager" This reverts commit 99cc86c46701c86f9b3a3188fce0d6dcba6c9592. --- stdlib/3/concurrent/futures/_base.pyi | 25 ++++++------------------- stdlib/3/concurrent/futures/process.pyi | 12 ++---------- stdlib/3/concurrent/futures/thread.pyi | 11 +++++------ 3 files changed, 13 insertions(+), 35 deletions(-) diff --git a/stdlib/3/concurrent/futures/_base.pyi b/stdlib/3/concurrent/futures/_base.pyi index 5e6c6f42d44a..836756d5d862 100644 --- a/stdlib/3/concurrent/futures/_base.pyi +++ b/stdlib/3/concurrent/futures/_base.pyi @@ -1,7 +1,5 @@ from typing import TypeVar, Generic, Any, Iterable, Iterator, Callable, Tuple, Optional, Set, NamedTuple -import sys - FIRST_COMPLETED = ... # type: str FIRST_EXCEPTION = ... # type: str ALL_COMPLETED = ... # type: str @@ -33,23 +31,12 @@ class Future(Generic[_T]): def set_result(self, result: _T) -> None: ... def set_exception(self, exception: Optional[BaseException]) -> None: ... - -if sys.version_info >= (3, 6): - from typing import ContextManager - _S = TypeVar('_S') - - class Executor(ContextManager[_S]): - def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... - def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... - def shutdown(self, wait: bool = ...) -> None: ... - -else: - class Executor: - def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... - def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... - def shutdown(self, wait: bool = ...) -> None: ... - def __enter__(self) -> Executor: ... - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool: ... +class Executor: + def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... + def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... + def shutdown(self, wait: bool = ...) -> None: ... + def __enter__(self) -> Executor: ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool: ... def as_completed(fs: Iterable[Future[_T]], timeout: Optional[float] = ...) -> Iterator[Future[_T]]: ... diff --git a/stdlib/3/concurrent/futures/process.pyi b/stdlib/3/concurrent/futures/process.pyi index f43ed703c8f6..8076c2d0588e 100644 --- a/stdlib/3/concurrent/futures/process.pyi +++ b/stdlib/3/concurrent/futures/process.pyi @@ -1,17 +1,9 @@ from typing import Optional, Any - -import sys - from ._base import Future, Executor EXTRA_QUEUED_CALLS = ... # type: Any class BrokenProcessPool(RuntimeError): ... -if sys.version_info >= (3,6): - class ProcessPoolExecutor(Executor[ProcessPoolExecutor]): - def __init__(self, max_workers: Optional[int] = ...) -> None: ... - -else: - class ProcessPoolExecutor(Executor): - def __init__(self, max_workers: Optional[int] = ...) -> None: ... +class ProcessPoolExecutor(Executor): + def __init__(self, max_workers: Optional[int] = ...) -> None: ... diff --git a/stdlib/3/concurrent/futures/thread.pyi b/stdlib/3/concurrent/futures/thread.pyi index f6dd78f018c6..88a45a3e070a 100644 --- a/stdlib/3/concurrent/futures/thread.pyi +++ b/stdlib/3/concurrent/futures/thread.pyi @@ -2,10 +2,9 @@ from typing import Optional from ._base import Executor, Future import sys -if sys.version_info >= (3, 6): - class ThreadPoolExecutor(Executor[ThreadPoolExecutor]): - def __init__(self, max_workers: Optional[int] = ..., thread_name_prefix: str = ...) -> None: ... - -else: - class ThreadPoolExecutor(Executor): +class ThreadPoolExecutor(Executor): + if sys.version_info >= (3, 6): + def __init__(self, max_workers: Optional[int] = ..., + thread_name_prefix: str = ...) -> None: ... + else: def __init__(self, max_workers: Optional[int] = ...) -> None: ... From 0ccc3f0832a50734c394d050c38c5832a7523b56 Mon Sep 17 00:00:00 2001 From: Josh Staiger Date: Sat, 4 Nov 2017 13:23:25 -0700 Subject: [PATCH 5/5] Make Executor.__enter__ self and return types match. Specifically, this solves a problem in code such as: with ThreadPoolExecutor() as p: ... Where the p variable would be typed as an abstract `Executor`, rather than the specific `ThreadPoolExecutor` as expected. --- stdlib/3/concurrent/futures/_base.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/3/concurrent/futures/_base.pyi b/stdlib/3/concurrent/futures/_base.pyi index 836756d5d862..56b119b37342 100644 --- a/stdlib/3/concurrent/futures/_base.pyi +++ b/stdlib/3/concurrent/futures/_base.pyi @@ -35,7 +35,7 @@ class Executor: def submit(self, fn: Callable[..., _T], *args: Any, **kwargs: Any) -> Future[_T]: ... def map(self, func: Callable[..., _T], *iterables: Iterable[Any], timeout: Optional[float] = ..., chunksize: int = ...) -> Iterator[_T]: ... def shutdown(self, wait: bool = ...) -> None: ... - def __enter__(self) -> Executor: ... + def __enter__(self: _T) -> _T: ... def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool: ... def as_completed(fs: Iterable[Future[_T]], timeout: Optional[float] = ...) -> Iterator[Future[_T]]: ...