Add plugin to infer more precise regex match types

This pull request adds a plugin to make mypy infer more
precise types when grabbing regex groups: the plugin will
when possible analyze original regex to deduce whether a
given group is required or not.

```
from typing_extensions import Final, Literal
import re

pattern: Final = re.compile("(a)(b)*")
match: Final = pattern.match("")
if match:
    reveal_type(match.groups())  # Revealed type is Tuple[str, Optional[str]]
    reveal_type(match.group(0))  # Revealed type is str
    reveal_type(match.group(1))  # Revealed type is str
    reveal_type(match.group(2))  # Revealed type is Optional[str]

    index: int
    reveal_type(match.group(index))  # Revealed type is Optional[str]

    # Error: Regex has 3 total groups, given group number 5 is too big
    match.group(5)
```

To track this information, I added in an optional 'metadata' dict
field to the Instance class, similar to the metadata dict for
plugins in TypeInfos. We skip serializing this dict if it does not
contain any data.

A limitation of this plugin is that both the pattern and the match
variables must be declared to be final. Otherwise, we just default
to using whatever types are defined in typeshed.

This is because we set and erase the metadata field in exactly the
same way we set and erase the `last_known_value` field in Instances:
both kinds of info are "transient" and are unsafe to keep around if
the variable reference is mutable.

This limitation *does* end up limiting the usefulness of this plugin
to some degree: it won't support common patterns like the below, since
variables aren't allowed to be declared final inside loops:

```
for line in file:
    match = pattern.match(line)
    if match:
        ...
```

Possibly we can remove this limitation by making mypy less aggressive
about removing this transient info by tracking the "lifetime" of this
sort of data in some way?

This pull request should mostly address
python#7363, though it's unclear if it
really fully resolves it: we might want to do something about the
limitation described above and re-tune typeshed first.

The other mostly unrelated change this PR makes is to refactor some of
the helper functions in checker.py into typeops.py so I could use them
more cleanly in the plugin.

Author: Michael0x2a

mypy/binder.py

@@ -10,7 +10,7 @@
 from mypy.subtypes import is_subtype
 from mypy.join import join_simple
 from mypy.sametypes import is_same_type
-from mypy.erasetype import remove_instance_last_known_values
+from mypy.erasetype import remove_instance_transient_info
 from mypy.nodes import Expression, Var, RefExpr
 from mypy.literals import Key, literal, literal_hash, subkeys
 from mypy.nodes import IndexExpr, MemberExpr, NameExpr
@@ -251,7 +251,7 @@ def assign_type(self, expr: Expression,
                     restrict_any: bool = False) -> None:
         # We should erase last known value in binder, because if we are using it,
         # it means that the target is not final, and therefore can't hold a literal.
-        type = remove_instance_last_known_values(type)
+        type = remove_instance_transient_info(type)
 
         type = get_proper_type(type)
         declared_type = get_proper_type(declared_type)

mypy/checker.py

@@ -2,7 +2,6 @@
 
 import itertools
 import fnmatch
-import sys
 from contextlib import contextmanager
 
 from typing import (
@@ -50,7 +49,8 @@
 from mypy.typeops import (
     map_type_from_supertype, bind_self, erase_to_bound, make_simplified_union,
     erase_def_to_union_or_bound, erase_to_union_or_bound,
-    true_only, false_only, function_type,
+    true_only, false_only, function_type, is_singleton_type,
+    try_expanding_enum_to_union, coerce_to_literal,
 )
 from mypy import message_registry
 from mypy.subtypes import (
@@ -63,7 +63,7 @@
 from mypy.typevars import fill_typevars, has_no_typevars, fill_typevars_with_any
 from mypy.semanal import set_callable_name, refers_to_fullname
 from mypy.mro import calculate_mro
-from mypy.erasetype import erase_typevars, remove_instance_last_known_values
+from mypy.erasetype import erase_typevars, remove_instance_transient_info
 from mypy.expandtype import expand_type, expand_type_by_instance
 from mypy.visitor import NodeVisitor
 from mypy.join import join_types
@@ -2069,7 +2069,7 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type
                         if partial_types is not None:
                             if not self.current_node_deferred:
                                 # Partial type can't be final, so strip any literal values.
-                                rvalue_type = remove_instance_last_known_values(rvalue_type)
+                                rvalue_type = remove_instance_transient_info(rvalue_type)
                                 inferred_type = make_simplified_union(
                                     [rvalue_type, NoneType()])
                                 self.set_inferred_type(var, lvalue, inferred_type)
@@ -2126,7 +2126,7 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type
             if inferred:
                 rvalue_type = self.expr_checker.accept(rvalue)
                 if not inferred.is_final:
-                    rvalue_type = remove_instance_last_known_values(rvalue_type)
+                    rvalue_type = remove_instance_transient_info(rvalue_type)
                 self.infer_variable_type(inferred, lvalue, rvalue_type, rvalue)
 
     def check_compatibility_all_supers(self, lvalue: RefExpr, lvalue_type: Optional[Type],
@@ -4753,97 +4753,6 @@ def is_private(node_name: str) -> bool:
     return node_name.startswith('__') and not node_name.endswith('__')
 
 
-def get_enum_values(typ: Instance) -> List[str]:
-    """Return the list of values for an Enum."""
-    return [name for name, sym in typ.type.names.items() if isinstance(sym.node, Var)]
-
-
-def is_singleton_type(typ: Type) -> bool:
-    """Returns 'true' if this type is a "singleton type" -- if there exists
-    exactly only one runtime value associated with this type.
-
-    That is, given two values 'a' and 'b' that have the same type 't',
-    'is_singleton_type(t)' returns True if and only if the expression 'a is b' is
-    always true.
-
-    Currently, this returns True when given NoneTypes, enum LiteralTypes and
-    enum types with a single value.
-
-    Note that other kinds of LiteralTypes cannot count as singleton types. For
-    example, suppose we do 'a = 100000 + 1' and 'b = 100001'. It is not guaranteed
-    that 'a is b' will always be true -- some implementations of Python will end up
-    constructing two distinct instances of 100001.
-    """
-    typ = get_proper_type(typ)
-    # TODO: Also make this return True if the type is a bool LiteralType.
-    # Also make this return True if the type corresponds to ... (ellipsis) or NotImplemented?
-    return (
-        isinstance(typ, NoneType) or (isinstance(typ, LiteralType) and typ.is_enum_literal())
-        or (isinstance(typ, Instance) and typ.type.is_enum and len(get_enum_values(typ)) == 1)
-    )
-
-
-def try_expanding_enum_to_union(typ: Type, target_fullname: str) -> ProperType:
-    """Attempts to recursively expand any enum Instances with the given target_fullname
-    into a Union of all of its component LiteralTypes.
-
-    For example, if we have:
-
-        class Color(Enum):
-            RED = 1
-            BLUE = 2
-            YELLOW = 3
-
-        class Status(Enum):
-            SUCCESS = 1
-            FAILURE = 2
-            UNKNOWN = 3
-
-    ...and if we call `try_expanding_enum_to_union(Union[Color, Status], 'module.Color')`,
-    this function will return Literal[Color.RED, Color.BLUE, Color.YELLOW, Status].
-    """
-    typ = get_proper_type(typ)
-
-    if isinstance(typ, UnionType):
-        items = [try_expanding_enum_to_union(item, target_fullname) for item in typ.items]
-        return make_simplified_union(items)
-    elif isinstance(typ, Instance) and typ.type.is_enum and typ.type.fullname() == target_fullname:
-        new_items = []
-        for name, symbol in typ.type.names.items():
-            if not isinstance(symbol.node, Var):
-                continue
-            new_items.append(LiteralType(name, typ))
-        # SymbolTables are really just dicts, and dicts are guaranteed to preserve
-        # insertion order only starting with Python 3.7. So, we sort these for older
-        # versions of Python to help make tests deterministic.
-        #
-        # We could probably skip the sort for Python 3.6 since people probably run mypy
-        # only using CPython, but we might as well for the sake of full correctness.
-        if sys.version_info < (3, 7):
-            new_items.sort(key=lambda lit: lit.value)
-        return make_simplified_union(new_items)
-    else:
-        return typ
-
-
-def coerce_to_literal(typ: Type) -> ProperType:
-    """Recursively converts any Instances that have a last_known_value or are
-    instances of enum types with a single value into the corresponding LiteralType.
-    """
-    typ = get_proper_type(typ)
-    if isinstance(typ, UnionType):
-        new_items = [coerce_to_literal(item) for item in typ.items]
-        return make_simplified_union(new_items)
-    elif isinstance(typ, Instance):
-        if typ.last_known_value:
-            return typ.last_known_value
-        elif typ.type.is_enum:
-            enum_values = get_enum_values(typ)
-            if len(enum_values) == 1:
-                return LiteralType(value=enum_values[0], fallback=typ)
-    return typ
-
-
 def has_bool_item(typ: ProperType) -> bool:
     """Return True if type is 'bool' or a union with a 'bool' item."""
     if is_named_instance(typ, 'builtins.bool'):

mypy/erasetype.py

@@ -123,15 +123,16 @@ def visit_type_var(self, t: TypeVarType) -> Type:
         return t
 
 
-def remove_instance_last_known_values(t: Type) -> Type:
-    return t.accept(LastKnownValueEraser())
-
+def remove_instance_transient_info(t: Type) -> Type:
+    """Recursively removes any info from Instances that exist
+    on a per-instance basis. Currently, this means erasing the
+    last-known literal type and any plugin metadata.
+    """
+    return t.accept(TransientInstanceInfoEraser())
 
-class LastKnownValueEraser(TypeTranslator):
-    """Removes the Literal[...] type that may be associated with any
-    Instance types."""
 
+class TransientInstanceInfoEraser(TypeTranslator):
     def visit_instance(self, t: Instance) -> Type:
-        if t.last_known_value:
-            return t.copy_modified(last_known_value=None)
+        if t.last_known_value or t.metadata:
+            return t.copy_modified(last_known_value=None, metadata={})
         return t

mypy/plugins/default.py

@@ -32,6 +32,8 @@ def get_function_hook(self, fullname: str
             return ctypes.array_constructor_callback
         elif fullname == 're.compile':
             return regex.re_compile_callback
+        elif fullname in regex.FUNCTIONS_PRODUCING_MATCH_OBJECT:
+            return regex.re_direct_match_callback
         return None
 
     def get_method_signature_hook(self, fullname: str
@@ -55,6 +57,7 @@ def get_method_signature_hook(self, fullname: str
     def get_method_hook(self, fullname: str
                         ) -> Optional[Callable[[MethodContext], Type]]:
         from mypy.plugins import ctypes
+        from mypy.plugins import regex
 
         if fullname == 'typing.Mapping.get':
             return typed_dict_get_callback
@@ -72,6 +75,12 @@ def get_method_hook(self, fullname: str
             return ctypes.array_iter_callback
         elif fullname == 'pathlib.Path.open':
             return path_open_callback
+        elif fullname in regex.METHODS_PRODUCING_MATCH_OBJECT:
+            return regex.re_get_match_callback
+        elif fullname == 'typing.Match.groups':
+            return regex.re_match_groups_callback
+        elif fullname in regex.METHODS_PRODUCING_GROUP:
+            return regex.re_match_group_callback
         return None
 
     def get_attribute_hook(self, fullname: str