pep612: add semanal for paramspec (#9339)

Co-authored-by: hauntsaninja <>

Author: hauntsaninja

mypy/checker.py

@@ -50,7 +50,7 @@
     erase_def_to_union_or_bound, erase_to_union_or_bound, coerce_to_literal,
     try_getting_str_literals_from_type, try_getting_int_literals_from_type,
     tuple_fallback, is_singleton_type, try_expanding_enum_to_union,
-    true_only, false_only, function_type, TypeVarExtractor, custom_special_method,
+    true_only, false_only, function_type, get_type_vars, custom_special_method,
     is_literal_type_like,
 )
 from mypy import message_registry
@@ -5328,7 +5328,7 @@ def detach_callable(typ: CallableType) -> CallableType:
 
     appear_map = {}  # type: Dict[str, List[int]]
     for i, inner_type in enumerate(type_list):
-        typevars_available = inner_type.accept(TypeVarExtractor())
+        typevars_available = get_type_vars(inner_type)
         for var in typevars_available:
             if var.fullname not in appear_map:
                 appear_map[var.fullname] = []
@@ -5338,7 +5338,7 @@ def detach_callable(typ: CallableType) -> CallableType:
     for var_name, appearances in appear_map.items():
         used_type_var_names.add(var_name)
 
-    all_type_vars = typ.accept(TypeVarExtractor())
+    all_type_vars = get_type_vars(typ)
     new_variables = []
     for var in set(all_type_vars):
         if var.fullname not in used_type_var_names:

mypy/checkexpr.py

@@ -31,6 +31,7 @@
     DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr,
     YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr,
     TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderNode,
+    ParamSpecExpr,
     ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, LITERAL_TYPE, REVEAL_TYPE,
 )
 from mypy.literals import literal
@@ -3973,6 +3974,9 @@ def visit_temp_node(self, e: TempNode) -> Type:
     def visit_type_var_expr(self, e: TypeVarExpr) -> Type:
         return AnyType(TypeOfAny.special_form)
 
+    def visit_paramspec_expr(self, e: ParamSpecExpr) -> Type:
+        return AnyType(TypeOfAny.special_form)
+
     def visit_newtype_expr(self, e: NewTypeExpr) -> Type:
         return AnyType(TypeOfAny.special_form)
 

mypy/literals.py

@@ -8,7 +8,7 @@
     ConditionalExpr, EllipsisExpr, YieldFromExpr, YieldExpr, RevealExpr, SuperExpr,
     TypeApplication, LambdaExpr, ListComprehension, SetComprehension, DictionaryComprehension,
     GeneratorExpr, BackquoteExpr, TypeVarExpr, TypeAliasExpr, NamedTupleExpr, EnumCallExpr,
-    TypedDictExpr, NewTypeExpr, PromoteExpr, AwaitExpr, TempNode, AssignmentExpr,
+    TypedDictExpr, NewTypeExpr, PromoteExpr, AwaitExpr, TempNode, AssignmentExpr, ParamSpecExpr
 )
 from mypy.visitor import ExpressionVisitor
 
@@ -213,6 +213,9 @@ def visit_backquote_expr(self, e: BackquoteExpr) -> None:
     def visit_type_var_expr(self, e: TypeVarExpr) -> None:
         return None
 
+    def visit_paramspec_expr(self, e: ParamSpecExpr) -> None:
+        return None
+
     def visit_type_alias_expr(self, e: TypeAliasExpr) -> None:
         return None
 

mypy/nodes.py

@@ -2043,23 +2043,10 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T:
 CONTRAVARIANT = 2  # type: Final[int]
 
 
-class TypeVarExpr(SymbolNode, Expression):
-    """Type variable expression TypeVar(...).
-
-    This is also used to represent type variables in symbol tables.
-
-    A type variable is not valid as a type unless bound in a TypeVarScope.
-    That happens within:
-
-     1. a generic class that uses the type variable as a type argument or
-     2. a generic function that refers to the type variable in its signature.
-    """
-
+class TypeVarLikeExpr(SymbolNode, Expression):
+    """Base class for TypeVarExpr and ParamSpecExpr."""
     _name = ''
     _fullname = ''
-    # Value restriction: only types in the list are valid as values. If the
-    # list is empty, there is no restriction.
-    values = None  # type: List[mypy.types.Type]
     # Upper bound: only subtypes of upper_bound are valid as values. By default
     # this is 'object', meaning no restriction.
     upper_bound = None  # type: mypy.types.Type
@@ -2069,14 +2056,12 @@ class TypeVarExpr(SymbolNode, Expression):
     # variable.
     variance = INVARIANT
 
-    def __init__(self, name: str, fullname: str,
-                 values: List['mypy.types.Type'],
-                 upper_bound: 'mypy.types.Type',
-                 variance: int = INVARIANT) -> None:
+    def __init__(
+        self, name: str, fullname: str, upper_bound: 'mypy.types.Type', variance: int = INVARIANT
+    ) -> None:
         super().__init__()
         self._name = name
         self._fullname = fullname
-        self.values = values
         self.upper_bound = upper_bound
         self.variance = variance
 
@@ -2088,6 +2073,29 @@ def name(self) -> str:
     def fullname(self) -> str:
         return self._fullname
 
+
+class TypeVarExpr(TypeVarLikeExpr):
+    """Type variable expression TypeVar(...).
+
+    This is also used to represent type variables in symbol tables.
+
+    A type variable is not valid as a type unless bound in a TypeVarScope.
+    That happens within:
+
+     1. a generic class that uses the type variable as a type argument or
+     2. a generic function that refers to the type variable in its signature.
+    """
+    # Value restriction: only types in the list are valid as values. If the
+    # list is empty, there is no restriction.
+    values = None  # type: List[mypy.types.Type]
+
+    def __init__(self, name: str, fullname: str,
+                 values: List['mypy.types.Type'],
+                 upper_bound: 'mypy.types.Type',
+                 variance: int = INVARIANT) -> None:
+        super().__init__(name, fullname, upper_bound, variance)
+        self.values = values
+
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
         return visitor.visit_type_var_expr(self)
 
@@ -2110,6 +2118,30 @@ def deserialize(cls, data: JsonDict) -> 'TypeVarExpr':
                            data['variance'])
 
 
+class ParamSpecExpr(TypeVarLikeExpr):
+    def accept(self, visitor: ExpressionVisitor[T]) -> T:
+        return visitor.visit_paramspec_expr(self)
+
+    def serialize(self) -> JsonDict:
+        return {
+            '.class': 'ParamSpecExpr',
+            'name': self._name,
+            'fullname': self._fullname,
+            'upper_bound': self.upper_bound.serialize(),
+            'variance': self.variance,
+        }
+
+    @classmethod
+    def deserialize(cls, data: JsonDict) -> 'ParamSpecExpr':
+        assert data['.class'] == 'ParamSpecExpr'
+        return ParamSpecExpr(
+            data['name'],
+            data['fullname'],
+            mypy.types.deserialize_type(data['upper_bound']),
+            data['variance']
+        )
+
+
 class TypeAliasExpr(Expression):
     """Type alias expression (rvalue)."""
 

mypy/semanal.py

@@ -76,6 +76,7 @@
     nongen_builtins, get_member_expr_fullname, REVEAL_TYPE,
     REVEAL_LOCALS, is_final_node, TypedDictExpr, type_aliases_target_versions,
     EnumCallExpr, RUNTIME_PROTOCOL_DECOS, FakeExpression, Statement, AssignmentExpr,
+    ParamSpecExpr
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.typevars import fill_typevars
@@ -1921,6 +1922,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         # * type variable definition
         elif self.process_typevar_declaration(s):
             special_form = True
+        elif self.process_paramspec_declaration(s):
+            special_form = True
         # * type constructors
         elif self.analyze_namedtuple_assign(s):
             special_form = True
@@ -2836,7 +2839,7 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool:
         Return True if this looks like a type variable declaration (but maybe
         with errors), otherwise return False.
         """
-        call = self.get_typevar_declaration(s)
+        call = self.get_typevarlike_declaration(s, ("typing.TypeVar",))
         if not call:
             return False
 
@@ -2847,7 +2850,7 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool:
             return False
 
         name = lvalue.name
-        if not self.check_typevar_name(call, name, s):
+        if not self.check_typevarlike_name(call, name, s):
             return False
 
         # Constraining types
@@ -2907,24 +2910,31 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> bool:
         self.add_symbol(name, call.analyzed, s)
         return True
 
-    def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool:
+    def check_typevarlike_name(self, call: CallExpr, name: str, context: Context) -> bool:
+        """Checks that the name of a TypeVar or ParamSpec matches its variable."""
         name = unmangle(name)
+        assert isinstance(call.callee, RefExpr)
+        typevarlike_type = (
+            call.callee.name if isinstance(call.callee, NameExpr) else call.callee.fullname
+        )
         if len(call.args) < 1:
-            self.fail("Too few arguments for TypeVar()", context)
+            self.fail("Too few arguments for {}()".format(typevarlike_type), context)
             return False
         if (not isinstance(call.args[0], (StrExpr, BytesExpr, UnicodeExpr))
                 or not call.arg_kinds[0] == ARG_POS):
-            self.fail("TypeVar() expects a string literal as first argument", context)
+            self.fail("{}() expects a string literal as first argument".format(typevarlike_type),
+                      context)
             return False
         elif call.args[0].value != name:
-            msg = "String argument 1 '{}' to TypeVar(...) does not match variable name '{}'"
-            self.fail(msg.format(call.args[0].value, name), context)
+            msg = "String argument 1 '{}' to {}(...) does not match variable name '{}'"
+            self.fail(msg.format(call.args[0].value, typevarlike_type, name), context)
             return False
         return True
 
-    def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]:
-        """Returns the TypeVar() call expression if `s` is a type var declaration
-        or None otherwise.
+    def get_typevarlike_declaration(self, s: AssignmentStmt,
+                                    typevarlike_types: Tuple[str, ...]) -> Optional[CallExpr]:
+        """Returns the call expression if `s` is a declaration of `typevarlike_type`
+        (TypeVar or ParamSpec), or None otherwise.
         """
         if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
             return None
@@ -2934,7 +2944,7 @@ def get_typevar_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]:
         callee = call.callee
         if not isinstance(callee, RefExpr):
             return None
-        if callee.fullname != 'typing.TypeVar':
+        if callee.fullname not in typevarlike_types:
             return None
         return call
 
@@ -3021,6 +3031,41 @@ def process_typevar_parameters(self, args: List[Expression],
             variance = INVARIANT
         return variance, upper_bound
 
+    def process_paramspec_declaration(self, s: AssignmentStmt) -> bool:
+        """Checks if s declares a ParamSpec; if yes, store it in symbol table.
+
+        Return True if this looks like a ParamSpec (maybe with errors), otherwise return False.
+
+        In the future, ParamSpec may accept bounds and variance arguments, in which
+        case more aggressive sharing of code with process_typevar_declaration should be pursued.
+        """
+        call = self.get_typevarlike_declaration(
+            s, ("typing_extensions.ParamSpec", "typing.ParamSpec")
+        )
+        if not call:
+            return False
+
+        lvalue = s.lvalues[0]
+        assert isinstance(lvalue, NameExpr)
+        if s.type:
+            self.fail("Cannot declare the type of a parameter specification", s)
+            return False
+
+        name = lvalue.name
+        if not self.check_typevarlike_name(call, name, s):
+            return False
+
+        # PEP 612 reserves the right to define bound, covariant and contravariant arguments to
+        # ParamSpec in a later PEP. If and when that happens, we should do something
+        # on the lines of process_typevar_parameters
+        paramspec_var = ParamSpecExpr(
+            name, self.qualified_name(name), self.object_type(), INVARIANT
+        )
+        paramspec_var.line = call.line
+        call.analyzed = paramspec_var
+        self.add_symbol(name, call.analyzed, s)
+        return True
+
     def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance) -> TypeInfo:
         class_def = ClassDef(name, Block([]))
         if self.is_func_scope() and not self.type:

mypy/strconv.py

@@ -467,6 +467,17 @@ def visit_type_var_expr(self, o: 'mypy.nodes.TypeVarExpr') -> str:
             a += ['UpperBound({})'.format(o.upper_bound)]
         return self.dump(a, o)
 
+    def visit_paramspec_expr(self, o: 'mypy.nodes.ParamSpecExpr') -> str:
+        import mypy.types
+        a = []  # type: List[Any]
+        if o.variance == mypy.nodes.COVARIANT:
+            a += ['Variance(COVARIANT)']
+        if o.variance == mypy.nodes.CONTRAVARIANT:
+            a += ['Variance(CONTRAVARIANT)']
+        if not mypy.types.is_named_instance(o.upper_bound, 'builtins.object'):
+            a += ['UpperBound({})'.format(o.upper_bound)]
+        return self.dump(a, o)
+
     def visit_type_alias_expr(self, o: 'mypy.nodes.TypeAliasExpr') -> str:
         return 'TypeAliasExpr({})'.format(o.type)
 

mypy/treetransform.py

@@ -15,7 +15,7 @@
     ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr,
     UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr,
     SliceExpr, OpExpr, UnaryExpr, LambdaExpr, TypeApplication, PrintStmt,
-    SymbolTable, RefExpr, TypeVarExpr, NewTypeExpr, PromoteExpr,
+    SymbolTable, RefExpr, TypeVarExpr, ParamSpecExpr, NewTypeExpr, PromoteExpr,
     ComparisonExpr, TempNode, StarExpr, Statement, Expression,
     YieldFromExpr, NamedTupleExpr, TypedDictExpr, NonlocalDecl, SetComprehension,
     DictionaryComprehension, ComplexExpr, TypeAliasExpr, EllipsisExpr,
@@ -498,6 +498,11 @@ def visit_type_var_expr(self, node: TypeVarExpr) -> TypeVarExpr:
                            self.types(node.values),
                            self.type(node.upper_bound), variance=node.variance)
 
+    def visit_paramspec_expr(self, node: ParamSpecExpr) -> ParamSpecExpr:
+        return ParamSpecExpr(
+            node.name, node.fullname, self.type(node.upper_bound), variance=node.variance
+        )
+
     def visit_type_alias_expr(self, node: TypeAliasExpr) -> TypeAliasExpr:
         return TypeAliasExpr(node.node)
 

mypy/visitor.py

@@ -155,6 +155,10 @@ def visit_backquote_expr(self, o: 'mypy.nodes.BackquoteExpr') -> T:
     def visit_type_var_expr(self, o: 'mypy.nodes.TypeVarExpr') -> T:
         pass
 
+    @abstractmethod
+    def visit_paramspec_expr(self, o: 'mypy.nodes.ParamSpecExpr') -> T:
+        pass
+
     @abstractmethod
     def visit_type_alias_expr(self, o: 'mypy.nodes.TypeAliasExpr') -> T:
         pass
@@ -529,6 +533,9 @@ def visit_backquote_expr(self, o: 'mypy.nodes.BackquoteExpr') -> T:
     def visit_type_var_expr(self, o: 'mypy.nodes.TypeVarExpr') -> T:
         pass
 
+    def visit_paramspec_expr(self, o: 'mypy.nodes.ParamSpecExpr') -> T:
+        pass
+
     def visit_type_alias_expr(self, o: 'mypy.nodes.TypeAliasExpr') -> T:
         pass
 

mypyc/irbuild/visitor.py

@@ -16,7 +16,7 @@
     FloatExpr, GeneratorExpr, GlobalDecl, LambdaExpr, ListComprehension, SetComprehension,
     NamedTupleExpr, NewTypeExpr, NonlocalDecl, OverloadedFuncDef, PrintStmt, RaiseStmt,
     RevealExpr, SetExpr, SliceExpr, StarExpr, SuperExpr, TryStmt, TypeAliasExpr, TypeApplication,
-    TypeVarExpr, TypedDictExpr, UnicodeExpr, WithStmt, YieldFromExpr, YieldExpr
+    TypeVarExpr, TypedDictExpr, UnicodeExpr, WithStmt, YieldFromExpr, YieldExpr, ParamSpecExpr
 )
 
 from mypyc.ir.ops import Value
@@ -309,6 +309,9 @@ def visit_type_application(self, o: TypeApplication) -> Value:
     def visit_type_var_expr(self, o: TypeVarExpr) -> Value:
         assert False, "can't compile analysis-only expressions"
 
+    def visit_paramspec_expr(self, o: ParamSpecExpr) -> Value:
+        assert False, "can't compile analysis-only expressions"
+
     def visit_typeddict_expr(self, o: TypedDictExpr) -> Value:
         assert False, "can't compile analysis-only expressions"
 

test-data/unit/lib-stub/typing.pyi

@@ -24,6 +24,7 @@ ClassVar = 0
 Final = 0
 NoReturn = 0
 NewType = 0
+ParamSpec = 0
 
 T = TypeVar('T')
 T_co = TypeVar('T_co', covariant=True)

test-data/unit/lib-stub/typing_extensions.pyi

@@ -21,6 +21,8 @@ Literal: _SpecialForm = ...
 
 Annotated: _SpecialForm = ...
 
+ParamSpec: _SpecialForm
+Concatenate: _SpecialForm
 
 # Fallback type for all typed dicts (does not exist at runtime).
 class _TypedDict(Mapping[str, object]):