Overhaul overload semantics, remove erasure, add union math

This pull request:

1.  Modifies how mypy handles overloads to match the proposal
    I made in the typing repo.

2.  Starts removing type erasure from overload checks.

3.  Adds support for basic union math.

4.  Makes overloads respect keyword-only args

This pull request does NOT implement the following:

1.  Special-casing descriptors

2.  Improving how operator methods are handled: I've tweaked one or two
    methods to take advantage of some of the changes I made in this PR,
    but stayed away from making any drastic changes for now.

3.  Detecting partially overlapping argument types -- e.g. code like:

        @overload
        def f(x: Union[A, B]) -> str: ...
        @overload
        def f(x: Union[B, C]) -> int: ...

        @overload
        def g(x: Tuple[int, ...]) -> str: ...
        @overload
        def g(x: Tuple[int, int]) -> int: ...

4.  Detecting overlapping "argument counts". For example, we should
    flag the following as an error since the first alternative could
    potentially overlap with the second.

        @overload
        def f(*args: int) -> int: ...
        @overload
        def f(x: int, y: int, z: int) -> str: ...

5.  The "is-more-precise" relation. It's working in most normal cases
    but still contains a few bugs, mostly relating to type vars.
    For example, this currently isn't being flagged as an error:

        class Wrapper(Generic[T]):
            @overload
            def f(self, x: int) -> int: ...  # No error?
            @overload
            def f(self, x: T) -> str: ...

    (This PR does the right thing if 'T' isn't bound to a containing
    class though:)

        class Wrapper:
            @overload
            def f(self, x: int, y: int) -> int: ...  # Error
            @overload
            def f(self, x: T, y: T) -> str: ...

    Currently, what I'm doing is using the existing `is_more_precise`
    method, which calls `is_proper_subtype`. I think i'll either need to
    rewrite `is_more_precise` to do what I want with typevars or find a
    way of forcing the two methods to unify their typevars before running
    the `is_proper_subtype` check.

The plan is to address these 5 TODOs in future pull requests. Items 1
and 2 are basically orthogonal to the overloads overhaul; items 3, 4,
and 5 basically boil down to finding ways to teach mypy to detect if one
thing is *potentially* compatible with another.

For example, mypy contains code to tell if one type is *definitely* a
subtype of another; fixing items 3 and 5 involve writing code to check
if a type is *potentially* a subtype of another.

Author: Michael0x2a

mypy/checker.py

@@ -39,7 +39,7 @@
 from mypy import messages
 from mypy.subtypes import (
     is_subtype, is_equivalent, is_proper_subtype, is_more_precise,
-    restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_subtype,
+    restrict_subtype_away, is_subtype_ignoring_tvars, is_callable_compatible,
     unify_generic_callable, find_member
 )
 from mypy.maptype import map_instance_to_supertype
@@ -407,22 +407,32 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
         if defn.info:
             self.check_method_override(defn)
             self.check_inplace_operator_method(defn)
-        self.check_overlapping_overloads(defn)
+        if not defn.is_property:
+            self.check_overlapping_overloads(defn)
         return None
 
     def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None:
         # At this point we should have set the impl already, and all remaining
         # items are decorators
         for i, item in enumerate(defn.items):
+            # TODO overloads involving decorators
             assert isinstance(item, Decorator)
             sig1 = self.function_type(item.func)
+
             for j, item2 in enumerate(defn.items[i + 1:]):
-                # TODO overloads involving decorators
                 assert isinstance(item2, Decorator)
                 sig2 = self.function_type(item2.func)
-                if is_unsafe_overlapping_signatures(sig1, sig2):
-                    self.msg.overloaded_signatures_overlap(i + 1, i + j + 2,
-                                                           item.func)
+
+                assert isinstance(sig1, CallableType)
+                assert isinstance(sig2, CallableType)
+
+                if not are_argument_counts_overlapping(sig1, sig2):
+                    continue
+
+                if if_overload_can_never_match(sig1, sig2):
+                    self.msg.overloaded_signature_will_never_match(i + 1, i + j + 2, item2.func)
+                elif is_unsafe_overlapping_overload_signatures(sig1, sig2):
+                    self.msg.overloaded_signatures_overlap(i + 1, i + j + 2, item.func)
             if defn.impl:
                 if isinstance(defn.impl, FuncDef):
                     impl_type = defn.impl.type
@@ -437,7 +447,8 @@ def check_overlapping_overloads(self, defn: OverloadedFuncDef) -> None:
 
                 assert isinstance(impl_type, CallableType)
                 assert isinstance(sig1, CallableType)
-                if not is_callable_subtype(impl_type, sig1, ignore_return=True):
+                if not is_callable_compatible(impl_type, sig1,
+                                              is_compat=is_subtype, ignore_return=True):
                     self.msg.overloaded_signatures_arg_specific(i + 1, defn.impl)
                 impl_type_subst = impl_type
                 if impl_type.variables:
@@ -1038,8 +1049,8 @@ def check_overlapping_op_methods(self,
                     fallback=self.named_type('builtins.function'),
                     name=reverse_type.name)
 
-                if is_unsafe_overlapping_signatures(forward_tweaked,
-                                                    reverse_tweaked):
+                if is_unsafe_overlapping_operator_signatures(
+                        forward_tweaked, reverse_tweaked):
                     self.msg.operator_method_signatures_overlap(
                         reverse_class, reverse_name,
                         forward_base, forward_name, context)
@@ -1832,10 +1843,18 @@ def check_multi_assignment_from_union(self, lvalues: List[Expression], rvalue: E
             # Bind a union of types collected in 'assignments' to every expression.
             if isinstance(expr, StarExpr):
                 expr = expr.expr
-            types, declared_types = zip(*items)
+
+            # TODO: See todo in binder.py, ConditionalTypeBinder.assign_type
+            # It's unclear why the 'declared_type' param is sometimes 'None'
+            clean_items = []  # type: List[Tuple[Type, Type]]
+            for type, declared_type in items:
+                assert declared_type is not None
+                clean_items.append((type, declared_type))
+
+            types, declared_types = zip(*clean_items)
             self.binder.assign_type(expr,
-                                    UnionType.make_simplified_union(types),
-                                    UnionType.make_simplified_union(declared_types),
+                                    UnionType.make_simplified_union(list(types)),
+                                    UnionType.make_simplified_union(list(declared_types)),
                                     False)
         for union, lv in zip(union_types, self.flatten_lvalues(lvalues)):
             # Properly store the inferred types.
@@ -3547,18 +3566,96 @@ def type(self, type: Type) -> Type:
         return expand_type(type, self.map)
 
 
-def is_unsafe_overlapping_signatures(signature: Type, other: Type) -> bool:
-    """Check if two signatures may be unsafely overlapping.
+def are_argument_counts_overlapping(t: CallableType, s: CallableType) -> bool:
+    """Can a single call match both t and s, based just on positional argument counts?
+    """
+    min_args = max(t.min_args, s.min_args)
+    max_args = min(t.max_possible_positional_args(), s.max_possible_positional_args())
+    return min_args <= max_args
 
-    Two signatures s and t are overlapping if both can be valid for the same
+
+def is_unsafe_overlapping_overload_signatures(signature: CallableType,
+                                              other: CallableType) -> bool:
+    """Check if two overloaded function signatures may be unsafely overlapping.
+
+    We consider two functions 's' and 't' to be unsafely overlapping both
+    of the following are true:
+
+    1.  s's parameters are all more precise or partially overlapping with t's
+    1.  s's return type is NOT a subtype of t's.
+
+    both can be valid for the same
     statically typed values and the return types are incompatible.
 
+    Assumes that 'signature' appears earlier in the list of overload
+    alternatives then 'other' and that their argument counts are overlapping.
+    """
+    # TODO: Handle partially overlapping parameter types and argument counts
+    #
+    # For example, the signatures "f(x: Union[A, B]) -> int" and "f(x: Union[B, C]) -> str"
+    # is unsafe: the parameter types are partially overlapping.
+    #
+    # To fix this, we need to either modify meet.is_overlapping_types or add a new
+    # function and use "is_more_precise(...) or is_partially_overlapping(...)" for the is_compat
+    # checks.
+    #
+    # Similarly, the signatures "f(x: A, y: A) -> str" and "f(*x: A) -> int" are also unsafe:
+    # the parameter *counts* or arity are partially overlapping.
+    #
+    # To fix this, we need to modify is_callable_compatible so it can optionally detect
+    # functions that are *potentially* compatible rather then *definitely* compatible.
+
+    # The reason we repeat this check twice is so we can do a slightly better job of
+    # checking for potentially overlapping param counts. Both calls will actually check
+    # the param and return types in the same "direction" -- the only thing that differs
+    # is how is_callable_compatible checks non-positional arguments.
+    return (is_callable_compatible(signature, other,
+                                   is_compat=is_more_precise,
+                                   is_compat_return=lambda l, r: not is_subtype(l, r),
+                                   check_args_covariantly=True) or
+            is_callable_compatible(other, signature,
+                                   is_compat=is_more_precise,
+                                   is_compat_return=lambda l, r: not is_subtype(r, l)))
+
+
+def if_overload_can_never_match(signature: CallableType, other: CallableType) -> bool:
+    """Check if the 'other' method can never be matched due to 'signature'.
+
+    This can happen if signature's parameters are all strictly broader then
+    other's parameters.
+
+    Assumes that both signatures have overlapping argument counts.
+    """
+    return is_callable_compatible(signature, other,
+                                  is_compat=is_more_precise,
+                                  ignore_return=True)
+
+
+def is_unsafe_overlapping_operator_signatures(signature: Type, other: Type) -> bool:
+    """Check if two operator method signatures may be unsafely overlapping.
+
+    Two signatures s and t are overlapping if both can be valid for the same
+        statically typed values and the return types are incompatible.
+
     Assume calls are first checked against 'signature', then against 'other'.
     Thus if 'signature' is more general than 'other', there is no unsafe
     overlapping.
 
-    TODO If argument types vary covariantly, the return type may vary
-         covariantly as well.
+    TODO: Clean up this function and make it not perform type erasure.
+
+    Context: This function was previously used to make sure both overloaded
+    functions and operator methods were not unsafely overlapping.
+
+    We changed the semantics for we should handle overloaded definitions,
+    but not operator functions. (We can't reuse the same semantics for both:
+    the overload semantics are too restrictive here).
+
+    We should rewrite this method so that:
+
+    1.  It uses many of the improvements made to overloads: in particular,
+        eliminating type erasure.
+
+    2.  It contains just the logic necessary for operator methods.
     """
     if isinstance(signature, CallableType):
         if isinstance(other, CallableType):
@@ -3601,12 +3698,11 @@ def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool:
     """Does t have wider arguments than s?"""
     # TODO should an overload with additional items be allowed to be more
     #      general than one with fewer items (or just one item)?
-    # TODO check argument kinds and otherwise make more general
     if isinstance(t, CallableType):
         if isinstance(s, CallableType):
-            t, s = unify_generic_callables(t, s)
-            return all(is_proper_subtype(args, argt)
-                       for argt, args in zip(t.arg_types, s.arg_types))
+            return is_callable_compatible(t, s,
+                                          is_compat=is_proper_subtype,
+                                          ignore_return=True)
     elif isinstance(t, FunctionLike):
         if isinstance(s, FunctionLike):
             if len(t.items()) == len(s.items()):
@@ -3615,29 +3711,6 @@ def is_more_general_arg_prefix(t: FunctionLike, s: FunctionLike) -> bool:
     return False
 
 
-def unify_generic_callables(t: CallableType,
-                            s: CallableType) -> Tuple[CallableType,
-                                                      CallableType]:
-    """Make type variables in generic callables the same if possible.
-
-    Return updated callables. If we can't unify the type variables,
-    return the unmodified arguments.
-    """
-    # TODO: Use this elsewhere when comparing generic callables.
-    if t.is_generic() and s.is_generic():
-        t_substitutions = {}
-        s_substitutions = {}
-        for tv1, tv2 in zip(t.variables, s.variables):
-            # Are these something we can unify?
-            if tv1.id != tv2.id and is_equivalent_type_var_def(tv1, tv2):
-                newdef = TypeVarDef.new_unification_variable(tv2)
-                t_substitutions[tv1.id] = TypeVarType(newdef)
-                s_substitutions[tv2.id] = TypeVarType(newdef)
-        return (cast(CallableType, expand_type(t, t_substitutions)),
-                cast(CallableType, expand_type(s, s_substitutions)))
-    return t, s
-
-
 def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool:
     """Are type variable definitions equivalent?
 
@@ -3653,17 +3726,17 @@ def is_equivalent_type_var_def(tv1: TypeVarDef, tv2: TypeVarDef) -> bool:
 
 
 def is_same_arg_prefix(t: CallableType, s: CallableType) -> bool:
-    # TODO check argument kinds
-    return all(is_same_type(argt, args)
-               for argt, args in zip(t.arg_types, s.arg_types))
+    return is_callable_compatible(t, s,
+                                  is_compat=is_same_type,
+                                  ignore_return=True,
+                                  check_args_covariantly=True,
+                                  ignore_pos_arg_names=True)
 
 
 def is_more_precise_signature(t: CallableType, s: CallableType) -> bool:
     """Is t more precise than s?
-
     A signature t is more precise than s if all argument types and the return
     type of t are more precise than the corresponding types in s.
-
     Assume that the argument kinds and names are compatible, and that the
     argument counts are overlapping.
     """