Skip to content

More precise return types for open(), Path.open(), bz2.open(), etc. #3371

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
May 28, 2020
Merged

More precise return types for open(), Path.open(), bz2.open(), etc. #3371

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
71506a8
Overload on mode for open() and Path.open()
ilai-deutel Oct 13, 2019
6bec337
Update Python2 stubs
ilai-deutel Oct 16, 2019
23749a6
overload + Literal on BZ2, gzip, LZMA
ilai-deutel Oct 16, 2019
952dc90
Apply black partially to builtin.pyi
ilai-deutel Oct 16, 2019
0335f3f
Fix pytype exceptions
ilai-deutel Oct 16, 2019
38e04fe
Move _OpenBinaryMode and _OpenTextMode to io; refactor code
ilai-deutel Oct 17, 2019
1619c9b
Fix missing import in io
ilai-deutel Oct 17, 2019
1eb282e
Merge branch 'master' into overloaded-open
ilai-deutel Oct 25, 2019
e6a4321
Merge branch 'master' into overloaded-open
ilai-deutel Nov 11, 2019
3a631c0
Merge branch 'master' into overloaded-open
ilai-deutel Nov 12, 2019
f924ef2
Merge branch 'master' into overloaded-open
ilai-deutel Jan 15, 2020
7455811
open() always return BinaryIO in Python2
ilai-deutel Jan 15, 2020
529d3b8
Merge branch 'master' into overloaded-open
ilai-deutel Apr 12, 2020
a4f11f1
Merge branch 'overloaded-open' of https://github.com/ilai-deutel/type…
JelleZijlstra May 28, 2020
70916ef
Merge remote-tracking branch 'upstream/master' into ilai-deutel-overl…
JelleZijlstra May 28, 2020
7bc89fe
Remove support for Python 3.4
ilai-deutel May 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions stdlib/2/__builtin__.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ from typing import (
TypeVar, Iterator, Iterable, NoReturn, overload, Container,
Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic,
Set, AbstractSet, FrozenSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs,
SupportsComplex, IO, BinaryIO, Union,
SupportsComplex, IO, BinaryIO, TextIO, Union,
ItemsView, KeysView, ValuesView, ByteString, Optional, AnyStr, Type, Text,
Protocol,
)
from abc import abstractmethod, ABCMeta
from ast import mod, AST
from io import _OpenBinaryMode, _OpenTextMode
from types import TracebackType, CodeType
import sys

Expand Down Expand Up @@ -1345,14 +1346,47 @@ def next(__i: Iterator[_T]) -> _T: ...
def next(__i: Iterator[_T], default: _VT) -> Union[_T, _VT]: ...
def oct(__i: Union[int, _SupportsIndex]) -> str: ...

if sys.version_info >= (3, 6):
def open(file: Union[str, bytes, int, _PathLike[Any]], mode: str = ..., buffering: int = ..., encoding: Optional[str] = ...,
errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any]: ...
elif sys.version_info >= (3,):
def open(file: Union[str, bytes, int], mode: str = ..., buffering: int = ..., encoding: Optional[str] = ...,
errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any]: ...
if sys.version_info >= (3,):
if sys.version_info >= (3, 6):
# Changed in version 3.6: Support added to accept objects implementing os.PathLike.
_OpenFile = Union[str, bytes, int, _PathLike[Any]]
else:
_OpenFile = Union[str, bytes, int]

@overload
def open(
file: _OpenFile,
mode: _OpenTextMode = ...,
buffering: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> TextIO: ...
@overload
def open(
file: _OpenFile,
mode: _OpenBinaryMode,
buffering: int = ...,
encoding: None = ...,
errors: None = ...,
newline: None = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> BinaryIO: ...
@overload
def open(
file: _OpenFile,
mode: str,
buffering: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> IO[Any]: ...

else:
def open(name: Union[unicode, int], mode: unicode = ..., buffering: int = ...) -> BinaryIO: ...

Expand Down
14 changes: 14 additions & 0 deletions stdlib/2/io.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# Only a subset of functionality is included.

from typing import List, BinaryIO, TextIO, IO, overload, Iterator, Iterable, Any, Union, Optional
from typing_extensions import Literal
import _io

from _io import BlockingIOError as BlockingIOError
Expand All @@ -21,6 +22,19 @@ from _io import TextIOWrapper as TextIOWrapper
from _io import UnsupportedOperation as UnsupportedOperation
from _io import open as open

_OpenTextMode = Literal[
'r', 'r+', '+r', 'rt', 'tr', 'rt+', 'r+t', '+rt', 'tr+', 't+r', '+tr',
'w', 'w+', '+w', 'wt', 'tw', 'wt+', 'w+t', '+wt', 'tw+', 't+w', '+tw',
'a', 'a+', '+a', 'at', 'ta', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta',
'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr',
]
_OpenBinaryMode = Literal[
'rb', 'br', 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
'wb', 'bw', 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
'ab', 'ba', 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr',
]

def _OpenWrapper(file: Union[str, unicode, int],
mode: unicode = ..., buffering: int = ..., encoding: unicode = ...,
errors: unicode = ..., newline: unicode = ...,
Expand Down
52 changes: 43 additions & 9 deletions stdlib/2and3/builtins.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ from typing import (
TypeVar, Iterator, Iterable, NoReturn, overload, Container,
Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic,
Set, AbstractSet, FrozenSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs,
SupportsComplex, IO, BinaryIO, Union,
SupportsComplex, IO, BinaryIO, TextIO, Union,
ItemsView, KeysView, ValuesView, ByteString, Optional, AnyStr, Type, Text,
Protocol,
)
from abc import abstractmethod, ABCMeta
from ast import mod, AST
from io import _OpenBinaryMode, _OpenTextMode
from types import TracebackType, CodeType
import sys

Expand Down Expand Up @@ -1345,14 +1346,47 @@ def next(__i: Iterator[_T]) -> _T: ...
def next(__i: Iterator[_T], default: _VT) -> Union[_T, _VT]: ...
def oct(__i: Union[int, _SupportsIndex]) -> str: ...

if sys.version_info >= (3, 6):
def open(file: Union[str, bytes, int, _PathLike[Any]], mode: str = ..., buffering: int = ..., encoding: Optional[str] = ...,
errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any]: ...
elif sys.version_info >= (3,):
def open(file: Union[str, bytes, int], mode: str = ..., buffering: int = ..., encoding: Optional[str] = ...,
errors: Optional[str] = ..., newline: Optional[str] = ..., closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...) -> IO[Any]: ...
if sys.version_info >= (3,):
if sys.version_info >= (3, 6):
# Changed in version 3.6: Support added to accept objects implementing os.PathLike.
_OpenFile = Union[str, bytes, int, _PathLike[Any]]
else:
_OpenFile = Union[str, bytes, int]

@overload
def open(
file: _OpenFile,
mode: _OpenTextMode = ...,
buffering: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> TextIO: ...
@overload
def open(
file: _OpenFile,
mode: _OpenBinaryMode,
buffering: int = ...,
encoding: None = ...,
errors: None = ...,
newline: None = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> BinaryIO: ...
@overload
def open(
file: _OpenFile,
mode: str,
buffering: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
closefd: bool = ...,
opener: Optional[Callable[[str, int], int]] = ...,
) -> IO[Any]: ...

else:
def open(name: Union[unicode, int], mode: unicode = ..., buffering: int = ...) -> BinaryIO: ...

Expand Down
62 changes: 44 additions & 18 deletions stdlib/2and3/bz2.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,58 @@
import io
import sys
from typing import Any, IO, Optional, Union
from os.path import _PathType
from typing import IO, Any, Optional, TextIO, Union, overload

if sys.version_info >= (3, 6):
from os import PathLike
_PathOrFile = Union[str, bytes, IO[Any], PathLike[Any]]
elif sys.version_info >= (3, 3):
_PathOrFile = Union[str, bytes, IO[Any]]
if sys.version_info >= (3, 8):
from typing import Literal
else:
_PathOrFile = str
from typing_extensions import Literal

_PathOrFile = Union[_PathType, IO[bytes]]

def compress(data: bytes, compresslevel: int = ...) -> bytes: ...
def decompress(data: bytes) -> bytes: ...

if sys.version_info >= (3, 3):
def open(filename: _PathOrFile,
mode: str = ...,
compresslevel: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...) -> IO[Any]: ...
if sys.version_info >= (3, 4):
# Changed in version 3.4: The 'x' (exclusive creation) mode was added.
_OpenBinaryMode = Literal["r", "rb", "w", "wb", "x", "xb", "a", "ab"]
_OpenTextMode = Literal["rt", "wt", "xt", "at"]
else:
_OpenBinaryMode = Literal["r", "rb", "w", "wb", "a", "ab"]
_OpenTextMode = Literal["rt", "wt", "at"]
@overload
def open(
filename: _PathOrFile,
mode: _OpenBinaryMode = ...,
compresslevel: int = ...,
encoding: None = ...,
errors: None = ...,
newline: None = ...,
) -> BZ2File: ...
@overload
def open(
filename: _PathType,
mode: _OpenTextMode,
compresslevel: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
) -> TextIO: ...
@overload
def open(
filename: _PathOrFile,
mode: str,
compresslevel: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
) -> Union[BZ2File, TextIO]: ...
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Union return types are usually problematic, since they force the use to assert or cast the return value, even if the user knows the return type. In this particular case I actually like the returned Union, since the last overload is only used if mode is dynamic, so type checking at runtime makes sence.


class BZ2File(io.BufferedIOBase, IO[bytes]): # type: ignore # python/mypy#5027
def __init__(self,
filename: _PathOrFile,
mode: str = ...,
buffering: Optional[Any] = ...,
compresslevel: int = ...) -> None: ...
def __init__(
self, filename: _PathOrFile, mode: str = ..., buffering: Optional[Any] = ..., compresslevel: int = ...
) -> None: ...

class BZ2Compressor(object):
def __init__(self, compresslevel: int = ...) -> None: ...
Expand Down
59 changes: 54 additions & 5 deletions stdlib/3/gzip.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,50 @@
from typing import Any, IO, Optional
from os.path import _PathType
import _compression
import sys
import zlib
from os.path import _PathType
from typing import IO, Optional, TextIO, Union, overload

import _compression

if sys.version_info >= (3, 8):
from typing import Literal
else:
from typing_extensions import Literal

if sys.version_info >= (3, 4):
# Changed in version 3.4: Added support for the 'x', 'xb' and 'xt' modes.
_OpenBinaryMode = Literal["r", "rb", "a", "ab", "w", "wb", "x", "xb"]
_OpenTextMode = Literal["rt", "at", "wt", "xt"]
else:
_OpenBinaryMode = Literal["r", "rb", "a", "ab", "w", "wb"]
_OpenTextMode = Literal["rt", "at", "wt"]

def open(filename, mode: str = ..., compresslevel: int = ..., encoding: Optional[str] = ..., errors: Optional[str] = ..., newline: Optional[str] = ...) -> IO[Any]: ...
@overload
def open(
filename: Union[_PathType, IO[bytes]],
mode: _OpenBinaryMode = ...,
compresslevel: int = ...,
encoding: None = ...,
errors: None = ...,
newline: None = ...,
) -> GzipFile: ...
@overload
def open(
filename: _PathType,
mode: _OpenTextMode,
compresslevel: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
) -> TextIO: ...
@overload
def open(
filename: Union[_PathType, IO[bytes]],
mode: str,
compresslevel: int = ...,
encoding: Optional[str] = ...,
errors: Optional[str] = ...,
newline: Optional[str] = ...,
) -> Union[GzipFile, TextIO]: ...

class _PaddedFile:
file: IO[bytes]
Expand All @@ -20,7 +60,14 @@ class GzipFile(_compression.BaseStream):
name: str
compress: zlib._Compress
fileobj: IO[bytes]
def __init__(self, filename: Optional[_PathType] = ..., mode: Optional[str] = ..., compresslevel: int = ..., fileobj: Optional[IO[bytes]] = ..., mtime: Optional[float] = ...) -> None: ...
def __init__(
self,
filename: Optional[_PathType] = ...,
mode: Optional[str] = ...,
compresslevel: int = ...,
fileobj: Optional[IO[bytes]] = ...,
mtime: Optional[float] = ...,
) -> None: ...
@property
def filename(self) -> str: ...
@property
Expand Down Expand Up @@ -48,6 +95,8 @@ class _GzipReader(_compression.DecompressReader):

if sys.version_info >= (3, 8):
def compress(data, compresslevel: int = ..., *, mtime: Optional[float] = ...) -> bytes: ...

else:
def compress(data, compresslevel: int = ...) -> bytes: ...

def decompress(data: bytes) -> bytes: ...
21 changes: 21 additions & 0 deletions stdlib/3/io.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,16 @@ from typing import (
)
import builtins
import codecs
import sys
from mmap import mmap
from types import TracebackType
from typing import TypeVar

if sys.version_info >= (3, 8):
from typing import Literal
else:
from typing_extensions import Literal

_bytearray_like = Union[bytearray, mmap]

DEFAULT_BUFFER_SIZE: int
Expand All @@ -17,6 +23,21 @@ SEEK_END: int

_T = TypeVar('_T', bound=IOBase)

_OpenTextMode = Literal[
'r', 'r+', '+r', 'rt', 'tr', 'rt+', 'r+t', '+rt', 'tr+', 't+r', '+tr',
'w', 'w+', '+w', 'wt', 'tw', 'wt+', 'w+t', '+wt', 'tw+', 't+w', '+tw',
'a', 'a+', '+a', 'at', 'ta', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta',
'x', 'x+', '+x', 'xt', 'tx', 'xt+', 'x+t', '+xt', 'tx+', 't+x', '+tx',
'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr',
]
_OpenBinaryMode = Literal[
'rb', 'br', 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br',
'wb', 'bw', 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw',
'ab', 'ba', 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba',
'xb', 'bx', 'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx',
'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr',
]

open = builtins.open

BlockingIOError = builtins.BlockingIOError
Expand Down
Loading