Make more type expressions valid in PEP 695 aliases and runtime contexts (#17404)

Previously some type expressions, when used as the value of a PEP 695
type alias or in an expression context, generated errors, even if the
code would work at runtime. Improve type inference of types in
expression contexts (this includes PEP 695 type aliases) to better
reflect runtime behavior.

This is still not perfect, since we don't have precise types for
everything in stubs. Use `typing._SpecialForm` as a fallback, as it
supports indexing and `|` operations, which are supported for types.

Also update stubs used in tests to better match typeshed stubs. In
particular, provide `_SpecialForm` and define `Any = object()`, similar
to typeshed.

Author: JukkaL

mypy/checkexpr.py

@@ -428,6 +428,9 @@ def analyze_var_ref(self, var: Var, context: Context) -> Type:
         if var.type:
             var_type = get_proper_type(var.type)
             if isinstance(var_type, Instance):
+                if var.fullname == "typing.Any":
+                    # The typeshed type is 'object'; give a more useful type in runtime context
+                    return self.named_type("typing._SpecialForm")
                 if self.is_literal_context() and var_type.last_known_value is not None:
                     return var_type.last_known_value
                 if var.name in {"True", "False"}:
@@ -4331,16 +4334,25 @@ def visit_index_with_type(
                 return self.nonliteral_tuple_index_helper(left_type, index)
         elif isinstance(left_type, TypedDictType):
             return self.visit_typeddict_index_expr(left_type, e.index)
-        elif (
-            isinstance(left_type, FunctionLike)
-            and left_type.is_type_obj()
-            and left_type.type_object().is_enum
-        ):
-            return self.visit_enum_index_expr(left_type.type_object(), e.index, e)
-        elif isinstance(left_type, TypeVarType) and not self.has_member(
+        elif isinstance(left_type, FunctionLike) and left_type.is_type_obj():
+            if left_type.type_object().is_enum:
+                return self.visit_enum_index_expr(left_type.type_object(), e.index, e)
+            elif left_type.type_object().type_vars:
+                return self.named_type("types.GenericAlias")
+            elif (
+                left_type.type_object().fullname == "builtins.type"
+                and self.chk.options.python_version >= (3, 9)
+            ):
+                # builtins.type is special: it's not generic in stubs, but it supports indexing
+                return self.named_type("typing._SpecialForm")
+
+        if isinstance(left_type, TypeVarType) and not self.has_member(
             left_type.upper_bound, "__getitem__"
         ):
             return self.visit_index_with_type(left_type.upper_bound, e, original_type)
+        elif isinstance(left_type, Instance) and left_type.type.fullname == "typing._SpecialForm":
+            # Allow special forms to be indexed and used to create union types
+            return self.named_type("typing._SpecialForm")
         else:
             result, method_type = self.check_method_call_by_name(
                 "__getitem__", left_type, [e.index], [ARG_POS], e, original_type=original_type

mypy/stubgenc.py

@@ -733,7 +733,7 @@ def generate_property_stub(
 
     def get_type_fullname(self, typ: type) -> str:
         """Given a type, return a string representation"""
-        if typ is Any:
+        if typ is Any:  # type: ignore[comparison-overlap]
             return "Any"
         typename = getattr(typ, "__qualname__", typ.__name__)
         module_name = self.get_obj_module(typ)

mypyc/test-data/fixtures/typing-full.pyi

@@ -15,8 +15,7 @@ class _SpecialForm:
 
 cast = 0
 overload = 0
-Any = 0
-Union = 0
+Any = object()
 Optional = 0
 TypeVar = 0
 Generic = 0
@@ -28,11 +27,12 @@ Type = 0
 no_type_check = 0
 ClassVar = 0
 Final = 0
-Literal = 0
 TypedDict = 0
 NoReturn = 0
 NewType = 0
 Callable: _SpecialForm
+Union: _SpecialForm
+Literal: _SpecialForm
 
 T = TypeVar('T')
 T_co = TypeVar('T_co', covariant=True)

test-data/unit/check-classes.test

@@ -4790,12 +4790,15 @@ def g(x: Type[S]) -> str:
     return reveal_type(x * 0)  # N: Revealed type is "builtins.str"
 
 [case testMetaclassGetitem]
+import types
+
 class M(type):
     def __getitem__(self, key) -> int: return 1
 
 class A(metaclass=M): pass
 
 reveal_type(A[M])  # N: Revealed type is "builtins.int"
+[builtins fixtures/tuple.pyi]
 
 [case testMetaclassSelfType]
 from typing import TypeVar, Type

test-data/unit/check-functions.test

@@ -1779,10 +1779,10 @@ def Arg(x, y): pass
 F = Callable[[Arg(int, 'x')], int]  # E: Invalid argument constructor "__main__.Arg"
 
 [case testCallableParsingFromExpr]
-
 from typing import Callable, List
 from mypy_extensions import Arg, VarArg, KwArg
 import mypy_extensions
+import types  # Needed for type checking
 
 def WrongArg(x, y): return y
 # Note that for this test, the 'Value of type "int" is not indexable' errors are silly,
@@ -1799,11 +1799,10 @@ L = Callable[[Arg(name='x', type=int)], int] # ok
 # I have commented out the following test because I don't know how to expect the "defined here" note part of the error.
 # M = Callable[[Arg(gnome='x', type=int)], int]   E: Invalid type alias: expression is not a valid type   E: Unexpected keyword argument "gnome" for "Arg"
 N = Callable[[Arg(name=None, type=int)], int] # ok
-O = Callable[[List[Arg(int)]], int]  # E: Invalid type alias: expression is not a valid type  # E: Value of type "int" is not indexable  # E: Type expected within [...] # E: The type "Type[List[Any]]" is not generic and not indexable
+O = Callable[[List[Arg(int)]], int]  # E: Invalid type alias: expression is not a valid type  # E: Value of type "int" is not indexable  # E: Type expected within [...]
 P = Callable[[mypy_extensions.VarArg(int)], int] # ok
 Q = Callable[[Arg(int, type=int)], int]  # E: Invalid type alias: expression is not a valid type  # E: Value of type "int" is not indexable  # E: "Arg" gets multiple values for keyword argument "type"
 R = Callable[[Arg(int, 'x', name='y')], int]  # E: Invalid type alias: expression is not a valid type  # E: Value of type "int" is not indexable  # E: "Arg" gets multiple values for keyword argument "name"
-
 [builtins fixtures/dict.pyi]
 
 [case testCallableParsing]

test-data/unit/check-generics.test

@@ -454,11 +454,13 @@ A[int, str, int]() # E: Type application has too many types (2 expected)
 [out]
 
 [case testInvalidTypeApplicationType]
+import types
 a: A
 class A: pass
 a[A]()  # E: Value of type "A" is not indexable
 A[A]()  # E: The type "Type[A]" is not generic and not indexable
-[out]
+[builtins fixtures/tuple.pyi]
+[typing fixtures/typing-full.pyi]
 
 [case testTypeApplicationArgTypes]
 from typing import TypeVar, Generic
@@ -513,8 +515,9 @@ Alias[int]("a")  # E: Argument 1 to "Node" has incompatible type "str"; expected
 [out]
 
 [case testTypeApplicationCrash]
+import types
 type[int] # this was crashing, see #2302 (comment)  # E: The type "Type[type]" is not generic and not indexable
-[out]
+[builtins fixtures/tuple.pyi]
 
 
 -- Generic type aliases

test-data/unit/check-python312.test

@@ -1591,3 +1591,55 @@ c: E[str]
 d: E[int]  # E: Type argument "int" of "E" must be a subtype of "str"
 [builtins fixtures/tuple.pyi]
 [typing fixtures/typing-full.pyi]
+
+[case testPEP695TypeAliasWithDifferentTargetTypes]
+# flags: --enable-incomplete-feature=NewGenericSyntax
+import types  # We need GenericAlias from here, and test stubs don't bring in 'types'
+from typing import Any, Callable, List, Literal, TypedDict
+
+# Test that various type expressions don't generate false positives as type alias
+# values, as they are type checked as expressions. There is a similar test case in
+# pythoneval.test that uses typeshed stubs.
+
+class C[T]: pass
+
+class TD(TypedDict):
+    x: int
+
+type A1 = type[int]
+type A2 = type[int] | None
+type A3 = None | type[int]
+type A4 = type[Any]
+
+type B1[**P, R] = Callable[P, R] | None
+type B2[**P, R] = None | Callable[P, R]
+type B3 = Callable[[str], int]
+type B4 = Callable[..., int]
+
+type C1 = A1 | None
+type C2 = None | A1
+
+type D1 = Any | None
+type D2 = None | Any
+
+type E1 = List[int]
+type E2 = List[int] | None
+type E3 = None | List[int]
+
+type F1 = Literal[1]
+type F2 = Literal['x'] | None
+type F3 = None | Literal[True]
+
+type G1 = tuple[int, Any]
+type G2 = tuple[int, Any] | None
+type G3 = None | tuple[int, Any]
+
+type H1 = TD
+type H2 = TD | None
+type H3 = None | TD
+
+type I1 = C[int]
+type I2 = C[Any] | None
+type I3 = None | C[TD]
+[builtins fixtures/type.pyi]
+[typing fixtures/typing-full.pyi]

test-data/unit/check-type-object-type-inference.test

@@ -2,6 +2,7 @@
 # flags: --python-version 3.9
 from typing import TypeVar, Generic, Type
 from abc import abstractmethod
+import types  # Explicitly bring in stubs for 'types'
 
 T = TypeVar('T')
 class E(Generic[T]):
@@ -37,5 +38,5 @@ def i(f: F):
     f.f(tuple[int,tuple[int,str]]).e( (27,(28,'z')) )  # OK
     reveal_type(f.f(tuple[int,tuple[int,str]]).e)  # N: Revealed type is "def (t: Tuple[builtins.int, Tuple[builtins.int, builtins.str]]) -> builtins.str"
 
-x = tuple[int,str][str]  # E: The type "Type[Tuple[Any, ...]]" is not generic and not indexable
+x = tuple[int,str][str]  # False negative
 [builtins fixtures/tuple.pyi]

test-data/unit/fixtures/typing-async.pyi

@@ -10,7 +10,7 @@ from abc import abstractmethod, ABCMeta
 
 cast = 0
 overload = 0
-Any = 0
+Any = object()
 Union = 0
 Optional = 0
 TypeVar = 0
@@ -125,3 +125,5 @@ class AsyncContextManager(Generic[T]):
     def __aenter__(self) -> Awaitable[T]: pass
     # Use Any because not all the precise types are in the fixtures.
     def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Awaitable[Any]: pass
+
+class _SpecialForm: pass

test-data/unit/fixtures/typing-full.pyi

@@ -12,6 +12,8 @@ class GenericMeta(type): pass
 
 class _SpecialForm:
     def __getitem__(self, index: Any) -> Any: ...
+    def __or__(self, other): ...
+    def __ror__(self, other): ...
 class TypeVar:
     def __init__(self, name, *args, bound=None): ...
     def __or__(self, other): ...
@@ -21,7 +23,7 @@ class TypeVarTuple: ...
 def cast(t, o): ...
 def assert_type(o, t): ...
 overload = 0
-Any = 0
+Any = object()
 Optional = 0
 Generic = 0
 Protocol = 0
@@ -31,14 +33,14 @@ Type = 0
 no_type_check = 0
 ClassVar = 0
 Final = 0
-Literal = 0
 TypedDict = 0
 NoReturn = 0
 NewType = 0
 Self = 0
 Unpack = 0
 Callable: _SpecialForm
 Union: _SpecialForm
+Literal: _SpecialForm
 
 T = TypeVar('T')
 T_co = TypeVar('T_co', covariant=True)
@@ -216,3 +218,4 @@ class TypeAliasType:
     ) -> None: ...
 
     def __or__(self, other: Any) -> Any: ...
+    def __ror__(self, other: Any) -> Any: ...

test-data/unit/fixtures/typing-medium.pyi

@@ -8,7 +8,7 @@
 
 cast = 0
 overload = 0
-Any = 0
+Any = object()
 Union = 0
 Optional = 0
 TypeVar = 0

test-data/unit/fixtures/typing-namedtuple.pyi

@@ -1,6 +1,6 @@
 TypeVar = 0
 Generic = 0
-Any = 0
+Any = object()
 overload = 0
 Type = 0
 Literal = 0
@@ -26,3 +26,5 @@ class NamedTuple(tuple[Any, ...]):
     def __init__(self, typename: str, fields: Iterable[tuple[str, Any]] = ...) -> None: ...
     @overload
     def __init__(self, typename: str, fields: None = None, **kwargs: Any) -> None: ...
+
+class _SpecialForm: pass

test-data/unit/fixtures/typing-override.pyi

@@ -1,6 +1,6 @@
 TypeVar = 0
 Generic = 0
-Any = 0
+Any = object()
 overload = 0
 Type = 0
 Literal = 0
@@ -21,5 +21,6 @@ class Mapping(Iterable[KT], Generic[KT, T_co]):
     def keys(self) -> Iterable[T]: pass  # Approximate return type
     def __getitem__(self, key: T) -> T_co: pass
 
-
 def override(__arg: T) -> T: ...
+
+class _SpecialForm: pass

test-data/unit/fixtures/typing-typeddict-iror.pyi

@@ -12,7 +12,7 @@ from abc import ABCMeta
 cast = 0
 assert_type = 0
 overload = 0
-Any = 0
+Any = object()
 Union = 0
 Optional = 0
 TypeVar = 0
@@ -64,3 +64,5 @@ class _TypedDict(Mapping[str, object]):
     def __ror__(self, __value: dict[str, Any]) -> dict[str, object]: ...
     # supposedly incompatible definitions of __or__ and __ior__
     def __ior__(self, __value: Self) -> Self: ...  # type: ignore[misc]
+
+class _SpecialForm: pass

test-data/unit/fixtures/typing-typeddict.pyi

@@ -11,7 +11,7 @@ from abc import ABCMeta
 cast = 0
 assert_type = 0
 overload = 0
-Any = 0
+Any = object()
 Union = 0
 Optional = 0
 TypeVar = 0
@@ -71,3 +71,5 @@ class _TypedDict(Mapping[str, object]):
     def pop(self, k: NoReturn, default: T = ...) -> object: ...
     def update(self: T, __m: T) -> None: ...
     def __delitem__(self, k: NoReturn) -> None: ...
+
+class _SpecialForm: pass

test-data/unit/lib-stub/types.pyi

@@ -9,7 +9,9 @@ class ModuleType:
     __file__: str
     def __getattr__(self, name: str) -> Any: pass
 
-class GenericAlias: ...
+class GenericAlias:
+    def __or__(self, o): ...
+    def __ror__(self, o): ...
 
 if sys.version_info >= (3, 10):
     class NoneType:

test-data/unit/lib-stub/typing.pyi

@@ -11,7 +11,7 @@
 cast = 0
 assert_type = 0
 overload = 0
-Any = 0
+Any = object()
 Union = 0
 Optional = 0
 TypeVar = 0
@@ -63,3 +63,5 @@ class Coroutine(Awaitable[V], Generic[T, U, V]): pass
 def final(meth: T) -> T: pass
 
 def reveal_type(__obj: T) -> T: pass
+
+class _SpecialForm: pass