From 357247843ef568a0ba254e43377e73afdbecdef5 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 28 Jul 2022 13:40:40 +0300 Subject: [PATCH 1/5] Remove all `builtins.unicode` references --- mypy/exprtotype.py | 4 +-- mypy/fastparse.py | 67 ++++++++---------------------------------- mypy/plugins/ctypes.py | 26 ++-------------- mypy/stubgen.py | 3 +- mypy/suggestions.py | 24 --------------- mypy/typeanal.py | 15 ++-------- mypy/types.py | 4 --- 7 files changed, 21 insertions(+), 122 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 6b96fa823c0b..e36e7f161a4c 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -169,11 +169,11 @@ def expr_to_unanalyzed_type( ) elif isinstance(expr, StrExpr): return parse_type_string( - expr.value, "builtins.str", expr.line, expr.column, assume_str_is_unicode=True + expr.value, "builtins.str", expr.line, expr.column ) elif isinstance(expr, BytesExpr): return parse_type_string( - expr.value, "builtins.bytes", expr.line, expr.column, assume_str_is_unicode=False + expr.value, "builtins.bytes", expr.line, expr.column ) elif isinstance(expr, UnaryExpr): typ = expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index d72447ecdba6..b8c1b87cbd62 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -335,7 +335,6 @@ def parse_type_comment( line: int, column: int, errors: Optional[Errors], - assume_str_is_unicode: bool = True, ) -> Tuple[Optional[List[str]], Optional[ProperType]]: """Parse type portion of a type comment (+ optional type ignore). @@ -369,7 +368,6 @@ def parse_type_comment( errors, line=line, override_column=column, - assume_str_is_unicode=assume_str_is_unicode, is_evaluated=False, ).visit(typ.body) return ignored, converted @@ -380,21 +378,11 @@ def parse_type_string( expr_fallback_name: str, line: int, column: int, - assume_str_is_unicode: bool = True, ) -> ProperType: - """Parses a type that was originally present inside of an explicit string, - byte string, or unicode string. + """Parses a type that was originally present inside of an explicit string. For example, suppose we have the type `Foo["blah"]`. We should parse the string expression "blah" using this function. - - If `assume_str_is_unicode` is set to true, this function will assume that - `Foo["blah"]` is equivalent to `Foo[u"blah"]`. Otherwise, it assumes it's - equivalent to `Foo[b"blah"]`. - - The caller is responsible for keeping track of the context in which the - type string was encountered (e.g. in Python 3 code, Python 2 code, Python 2 - code with unicode_literals...) and setting `assume_str_is_unicode` accordingly. """ try: _, node = parse_type_comment( @@ -402,7 +390,6 @@ def parse_type_string( line=line, column=column, errors=None, - assume_str_is_unicode=assume_str_is_unicode, ) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string @@ -1743,14 +1730,12 @@ def __init__( errors: Optional[Errors], line: int = -1, override_column: int = -1, - assume_str_is_unicode: bool = True, is_evaluated: bool = True, ) -> None: self.errors = errors self.line = line self.override_column = override_column self.node_stack: List[AST] = [] - self.assume_str_is_unicode = assume_str_is_unicode self.is_evaluated = is_evaluated def convert_column(self, column: int) -> int: @@ -1921,22 +1906,12 @@ def visit_Constant(self, n: Constant) -> Type: return UnboundType("None", line=self.line) if isinstance(val, str): # Parse forward reference. - if (n.kind and "u" in n.kind) or self.assume_str_is_unicode: - return parse_type_string( - n.s, - "builtins.unicode", - self.line, - n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode, - ) - else: - return parse_type_string( - n.s, - "builtins.str", - self.line, - n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode, - ) + return parse_type_string( + n.s, + "builtins.str", + self.line, + n.col_offset, + ) if val is Ellipsis: # '...' is valid in some types. return EllipsisType(line=self.line) @@ -1990,34 +1965,18 @@ def visit_Num(self, n: Num) -> Type: # Str(string s) def visit_Str(self, n: Str) -> Type: - # Note: we transform these fallback types into the correct types in - # 'typeanal.py' -- specifically in the named_type_with_normalized_str method. - # If we're analyzing Python 3, that function will translate 'builtins.unicode' - # into 'builtins.str'. In contrast, if we're analyzing Python 2 code, we'll - # translate 'builtins.bytes' in the method below into 'builtins.str'. - # Do a getattr because the field doesn't exist in 3.8 (where # this method doesn't actually ever run.) We can't just do # an attribute access with a `# type: ignore` because it would be # unused on < 3.8. kind: str = getattr(n, "kind") # noqa - if "u" in kind or self.assume_str_is_unicode: - return parse_type_string( - n.s, - "builtins.unicode", - self.line, - n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode, - ) - else: - return parse_type_string( - n.s, - "builtins.str", - self.line, - n.col_offset, - assume_str_is_unicode=self.assume_str_is_unicode, - ) + return parse_type_string( + n.s, + "builtins.str", + self.line, + n.col_offset, + ) # Bytes(bytes s) def visit_Bytes(self, n: Bytes) -> Type: diff --git a/mypy/plugins/ctypes.py b/mypy/plugins/ctypes.py index b2a12cc7ba1a..64138d136245 100644 --- a/mypy/plugins/ctypes.py +++ b/mypy/plugins/ctypes.py @@ -23,26 +23,6 @@ ) -def _get_bytes_type(api: "mypy.plugin.CheckerPluginInterface") -> Instance: - """Return the type corresponding to bytes on the current Python version. - - This is bytes in Python 3, and str in Python 2. - """ - return api.named_generic_type( - "builtins.bytes" if api.options.python_version >= (3,) else "builtins.str", [] - ) - - -def _get_text_type(api: "mypy.plugin.CheckerPluginInterface") -> Instance: - """Return the type corresponding to Text on the current Python version. - - This is str in Python 3, and unicode in Python 2. - """ - return api.named_generic_type( - "builtins.str" if api.options.python_version >= (3,) else "builtins.unicode", [] - ) - - def _find_simplecdata_base_arg( tp: Instance, api: "mypy.plugin.CheckerPluginInterface" ) -> Optional[ProperType]: @@ -221,9 +201,9 @@ def array_value_callback(ctx: "mypy.plugin.AttributeContext") -> Type: if isinstance(tp, AnyType): types.append(AnyType(TypeOfAny.from_another_any, source_any=tp)) elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_char": - types.append(_get_bytes_type(ctx.api)) + types.append(ctx.api.named_generic_type("builtins.bytes")) elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_wchar": - types.append(_get_text_type(ctx.api)) + types.append(ctx.api.named_generic_type("builtins.str")) else: ctx.api.msg.fail( 'Array attribute "value" is only available' @@ -245,7 +225,7 @@ def array_raw_callback(ctx: "mypy.plugin.AttributeContext") -> Type: or isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_char" ): - types.append(_get_bytes_type(ctx.api)) + types.append(ctx.api.named_generic_type("builtins.bytes")) else: ctx.api.msg.fail( 'Array attribute "raw" is only available' diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 08f86d96be11..8528c9b14e77 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -308,11 +308,10 @@ def args_str(self, args: Iterable[Type]) -> str: The main difference from list_str is the preservation of quotes for string arguments """ - types = ["builtins.bytes", "builtins.unicode"] res = [] for arg in args: arg_str = arg.accept(self) - if isinstance(arg, UnboundType) and arg.original_str_fallback in types: + if isinstance(arg, UnboundType) and arg.original_str_fallback == "builtins.bytes": res.append(f"'{arg_str}'") else: res.append(arg_str) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 3829fc26b84a..5d4c200aeb59 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -401,12 +401,6 @@ def get_default_arg_types(self, fdef: FuncDef) -> List[Optional[Type]]: for arg in fdef.arguments ] - def add_adjustments(self, typs: List[Type]) -> List[Type]: - if not self.try_text or self.manager.options.python_version[0] != 2: - return typs - translator = StrToText(self.named_type) - return dedup(typs + [tp.accept(translator) for tp in typs]) - def get_guesses( self, is_method: bool, @@ -420,7 +414,6 @@ def get_guesses( This focuses just on the argument types, and doesn't change the provided return type. """ options = self.get_args(is_method, base, defaults, callsites, uses) - options = [self.add_adjustments(tps) for tps in options] # Take the first `max_guesses` guesses. product = itertools.islice(itertools.product(*options), 0, self.max_guesses) @@ -909,23 +902,6 @@ def visit_callable_type(self, t: CallableType) -> str: return f"Callable[{arg_str}, {t.ret_type.accept(self)}]" -class StrToText(TypeTranslator): - def __init__(self, named_type: Callable[[str], Instance]) -> None: - self.text_type = named_type("builtins.unicode") - - def visit_type_alias_type(self, t: TypeAliasType) -> Type: - exp_t = get_proper_type(t) - if isinstance(exp_t, Instance) and exp_t.type.fullname == "builtins.str": - return self.text_type - return t.copy_modified(args=[a.accept(self) for a in t.args]) - - def visit_instance(self, t: Instance) -> Type: - if t.type.fullname == "builtins.str": - return self.text_type - else: - return super().visit_instance(t) - - TType = TypeVar("TType", bound=Type) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 78cfb8b59935..d6615d4f4c9e 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1162,7 +1162,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L return [ LiteralType( value=arg.original_str_expr, - fallback=self.named_type_with_normalized_str(arg.original_str_fallback), + fallback=self.named_type(arg.original_str_fallback), line=arg.line, column=arg.column, ) @@ -1210,7 +1210,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L return None # Remap bytes and unicode into the appropriate type for the correct Python version - fallback = self.named_type_with_normalized_str(arg.base_type_name) + fallback = self.named_type(arg.base_type_name) assert isinstance(fallback, Instance) return [LiteralType(arg.literal_value, fallback, line=arg.line, column=arg.column)] elif isinstance(arg, (NoneType, LiteralType)): @@ -1357,17 +1357,6 @@ def anal_var_def(self, var_def: TypeVarLikeType) -> TypeVarLikeType: def anal_var_defs(self, var_defs: Sequence[TypeVarLikeType]) -> List[TypeVarLikeType]: return [self.anal_var_def(vd) for vd in var_defs] - def named_type_with_normalized_str(self, fully_qualified_name: str) -> Instance: - """Does almost the same thing as `named_type`, except that we immediately - unalias `builtins.bytes` and `builtins.unicode` to `builtins.str` as appropriate. - """ - python_version = self.options.python_version - if python_version[0] == 2 and fully_qualified_name == "builtins.bytes": - fully_qualified_name = "builtins.str" - if python_version[0] >= 3 and fully_qualified_name == "builtins.unicode": - fully_qualified_name = "builtins.str" - return self.named_type(fully_qualified_name) - def named_type( self, fully_qualified_name: str, diff --git a/mypy/types.py b/mypy/types.py index 5d830d8091d6..ad39edee4112 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2406,10 +2406,6 @@ def value_repr(self) -> str: # Note: 'builtins.bytes' only appears in Python 3, so we want to # explicitly prefix with a "b" return "b" + raw - elif fallback_name == "builtins.unicode": - # Similarly, 'builtins.unicode' only appears in Python 2, where we also - # want to explicitly prefix - return "u" + raw else: # 'builtins.str' could mean either depending on context, but either way # we don't prefix: it's the "native" string. And of course, if value is From d93fba58a4948296f2720d35966b45aab475369c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 28 Jul 2022 13:48:03 +0300 Subject: [PATCH 2/5] Fix CI --- mypy/plugins/ctypes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/ctypes.py b/mypy/plugins/ctypes.py index 64138d136245..357b28a48385 100644 --- a/mypy/plugins/ctypes.py +++ b/mypy/plugins/ctypes.py @@ -201,9 +201,9 @@ def array_value_callback(ctx: "mypy.plugin.AttributeContext") -> Type: if isinstance(tp, AnyType): types.append(AnyType(TypeOfAny.from_another_any, source_any=tp)) elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_char": - types.append(ctx.api.named_generic_type("builtins.bytes")) + types.append(ctx.api.named_generic_type("builtins.bytes", [])) elif isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_wchar": - types.append(ctx.api.named_generic_type("builtins.str")) + types.append(ctx.api.named_generic_type("builtins.str", [])) else: ctx.api.msg.fail( 'Array attribute "value" is only available' @@ -225,7 +225,7 @@ def array_raw_callback(ctx: "mypy.plugin.AttributeContext") -> Type: or isinstance(tp, Instance) and tp.type.fullname == "ctypes.c_char" ): - types.append(ctx.api.named_generic_type("builtins.bytes")) + types.append(ctx.api.named_generic_type("builtins.bytes", [])) else: ctx.api.msg.fail( 'Array attribute "raw" is only available' From d37c449065f558679b7cc903e1c077d6c859765e Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 28 Jul 2022 13:53:02 +0300 Subject: [PATCH 3/5] black --- mypy/exprtotype.py | 8 ++------ mypy/fastparse.py | 36 ++++++------------------------------ 2 files changed, 8 insertions(+), 36 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index e36e7f161a4c..76fe2d511762 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -168,13 +168,9 @@ def expr_to_unanalyzed_type( column=expr.column, ) elif isinstance(expr, StrExpr): - return parse_type_string( - expr.value, "builtins.str", expr.line, expr.column - ) + return parse_type_string(expr.value, "builtins.str", expr.line, expr.column) elif isinstance(expr, BytesExpr): - return parse_type_string( - expr.value, "builtins.bytes", expr.line, expr.column - ) + return parse_type_string(expr.value, "builtins.bytes", expr.line, expr.column) elif isinstance(expr, UnaryExpr): typ = expr_to_unanalyzed_type(expr.expr, options, allow_new_syntax) if isinstance(typ, RawExpressionType): diff --git a/mypy/fastparse.py b/mypy/fastparse.py index b8c1b87cbd62..f672ab7ff707 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -331,10 +331,7 @@ def parse_type_ignore_tag(tag: Optional[str]) -> Optional[List[str]]: def parse_type_comment( - type_comment: str, - line: int, - column: int, - errors: Optional[Errors], + type_comment: str, line: int, column: int, errors: Optional[Errors] ) -> Tuple[Optional[List[str]], Optional[ProperType]]: """Parse type portion of a type comment (+ optional type ignore). @@ -365,19 +362,13 @@ def parse_type_comment( ignored = None assert isinstance(typ, ast3_Expression) converted = TypeConverter( - errors, - line=line, - override_column=column, - is_evaluated=False, + errors, line=line, override_column=column, is_evaluated=False ).visit(typ.body) return ignored, converted def parse_type_string( - expr_string: str, - expr_fallback_name: str, - line: int, - column: int, + expr_string: str, expr_fallback_name: str, line: int, column: int ) -> ProperType: """Parses a type that was originally present inside of an explicit string. @@ -385,12 +376,7 @@ def parse_type_string( string expression "blah" using this function. """ try: - _, node = parse_type_comment( - expr_string.strip(), - line=line, - column=column, - errors=None, - ) + _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string node.original_str_fallback = expr_fallback_name @@ -1906,12 +1892,7 @@ def visit_Constant(self, n: Constant) -> Type: return UnboundType("None", line=self.line) if isinstance(val, str): # Parse forward reference. - return parse_type_string( - n.s, - "builtins.str", - self.line, - n.col_offset, - ) + return parse_type_string(n.s, "builtins.str", self.line, n.col_offset) if val is Ellipsis: # '...' is valid in some types. return EllipsisType(line=self.line) @@ -1971,12 +1952,7 @@ def visit_Str(self, n: Str) -> Type: # unused on < 3.8. kind: str = getattr(n, "kind") # noqa - return parse_type_string( - n.s, - "builtins.str", - self.line, - n.col_offset, - ) + return parse_type_string(n.s, "builtins.str", self.line, n.col_offset) # Bytes(bytes s) def visit_Bytes(self, n: Bytes) -> Type: From 6a91d5c347d08dbe42e23fb34ae1750a091f86c1 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 28 Jul 2022 15:12:36 +0300 Subject: [PATCH 4/5] Fix ci --- mypy/stubgen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 8528c9b14e77..62fd685e2777 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -308,10 +308,11 @@ def args_str(self, args: Iterable[Type]) -> str: The main difference from list_str is the preservation of quotes for string arguments """ + types = ["builtins.bytes", "builtins.str"] res = [] for arg in args: arg_str = arg.accept(self) - if isinstance(arg, UnboundType) and arg.original_str_fallback == "builtins.bytes": + if isinstance(arg, UnboundType) and arg.original_str_fallback in types: res.append(f"'{arg_str}'") else: res.append(arg_str) From 671f5c24655b68d8310dcd4bde8f13320a77fdb6 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 28 Jul 2022 18:46:19 +0300 Subject: [PATCH 5/5] Address review --- docs/source/mypy_daemon.rst | 5 ----- mypy/dmypy/client.py | 4 ---- mypy/fastparse.py | 6 ------ mypy/suggestions.py | 4 ---- mypy/test/testfinegrained.py | 2 -- 5 files changed, 21 deletions(-) diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst index ce4f1582558c..503af805779c 100644 --- a/docs/source/mypy_daemon.rst +++ b/docs/source/mypy_daemon.rst @@ -228,11 +228,6 @@ command. Only allow some fraction of types in the suggested signature to be ``Any`` types. The fraction ranges from ``0`` (same as ``--no-any``) to ``1``. -.. option:: --try-text - - Try also using ``unicode`` wherever ``str`` is inferred. This flag may be useful - for annotating Python 2/3 straddling code. - .. option:: --callsites Only find call sites for a given function instead of suggesting a type. diff --git a/mypy/dmypy/client.py b/mypy/dmypy/client.py index d67f25d0d9b4..027f9ea2d515 100644 --- a/mypy/dmypy/client.py +++ b/mypy/dmypy/client.py @@ -161,9 +161,6 @@ def __init__(self, prog: str) -> None: type=float, help="Allow anys in types if they go above a certain score (scores are from 0-1)", ) -p.add_argument( - "--try-text", action="store_true", help="Try using unicode wherever str is inferred" -) p.add_argument( "--callsites", action="store_true", help="Find callsites instead of suggesting a type" ) @@ -525,7 +522,6 @@ def do_suggest(args: argparse.Namespace) -> None: no_errors=args.no_errors, no_any=args.no_any, flex_any=args.flex_any, - try_text=args.try_text, use_fixme=args.use_fixme, max_guesses=args.max_guesses, ) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index f672ab7ff707..b3cf7bdaf47f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1946,12 +1946,6 @@ def visit_Num(self, n: Num) -> Type: # Str(string s) def visit_Str(self, n: Str) -> Type: - # Do a getattr because the field doesn't exist in 3.8 (where - # this method doesn't actually ever run.) We can't just do - # an attribute access with a `# type: ignore` because it would be - # unused on < 3.8. - kind: str = getattr(n, "kind") # noqa - return parse_type_string(n.s, "builtins.str", self.line, n.col_offset) # Bytes(bytes s) diff --git a/mypy/suggestions.py b/mypy/suggestions.py index 5d4c200aeb59..a5ff07da3ba4 100644 --- a/mypy/suggestions.py +++ b/mypy/suggestions.py @@ -249,7 +249,6 @@ def __init__( json: bool, no_errors: bool = False, no_any: bool = False, - try_text: bool = False, flex_any: Optional[float] = None, use_fixme: Optional[str] = None, max_guesses: Optional[int] = None, @@ -262,7 +261,6 @@ def __init__( self.give_json = json self.no_errors = no_errors - self.try_text = try_text self.flex_any = flex_any if no_any: self.flex_any = 1.0 @@ -768,8 +766,6 @@ def score_type(self, t: Type, arg_pos: bool) -> int: return 10 if isinstance(t, CallableType) and (has_any_type(t) or is_tricky_callable(t)): return 10 - if self.try_text and isinstance(t, Instance) and t.type.fullname == "builtins.str": - return 1 return 0 def score_callable(self, t: CallableType) -> int: diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 61a5b90d7421..faf2a9ecc171 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -292,7 +292,6 @@ def maybe_suggest(self, step: int, server: Server, src: str, tmp_dir: str) -> Li callsites = "--callsites" in flags no_any = "--no-any" in flags no_errors = "--no-errors" in flags - try_text = "--try-text" in flags m = re.match("--flex-any=([0-9.]+)", flags) flex_any = float(m.group(1)) if m else None m = re.match(r"--use-fixme=(\w+)", flags) @@ -304,7 +303,6 @@ def maybe_suggest(self, step: int, server: Server, src: str, tmp_dir: str) -> Li json=json, no_any=no_any, no_errors=no_errors, - try_text=try_text, flex_any=flex_any, use_fixme=use_fixme, callsites=callsites,