From 965531f1fb174983a29beb42264d381dc1638ba4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 25 Jul 2016 13:39:12 -0700 Subject: [PATCH 01/26] Add NewVarExpr node --- mypy/nodes.py | 29 +++++++++++++++++++++++++++++ mypy/visitor.py | 3 +++ 2 files changed, 32 insertions(+) diff --git a/mypy/nodes.py b/mypy/nodes.py index cc77c8b82c57..2c4c2f6a892d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1710,6 +1710,35 @@ def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit__promote_expr(self) +class NewTypeExpr(SymbolNode, Expression): + """NewType expression NewType(...).""" + + name = '' + fullname = '' + value = None # type: mypy.types.Type + + def __init__(self, name: str, fullname: str, value: 'mypy.type.Type') -> None: + self.name = name + self.fullname = fullname + self.value = value + + def accept(self, visitor: NodeVisitor[T]) -> T: + return visitor.visit_newtype_expr(self) + + def serialize(self) -> JsonDict: + return {'.class': 'NewTypeExpr', + 'name': self.name, + 'fullname': self.fullname, + 'value': self.value.serialize()} + + @classmethod + def deserialize(cls, data, JsonDict) -> 'TypeVarExpr': + assert data['.class'] == 'NewTypeExpr' + return NewTypeExpr(data['name'], + data['fullname'], + mypy.types.Type.deserialize(data['value'])) + + class AwaitExpr(Node): """Await expression (await ...).""" diff --git a/mypy/visitor.py b/mypy/visitor.py index 43e7c161ea6d..b4c2cc86038b 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -225,6 +225,9 @@ def visit_type_alias_expr(self, o: 'mypy.nodes.TypeAliasExpr') -> T: def visit_namedtuple_expr(self, o: 'mypy.nodes.NamedTupleExpr') -> T: pass + def visit_newtype_expr(self, o: 'mypy.nodes.NewTypeExpr') -> T: + pass + def visit__promote_expr(self, o: 'mypy.nodes.PromoteExpr') -> T: pass From 59471ce5dd90e4a28b557eaa900dc0a70f08de37 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 25 Jul 2016 16:53:18 -0700 Subject: [PATCH 02/26] Modify NameExpr, get prelim version working --- mypy/checker.py | 5 ++- mypy/fixup.py | 5 ++- mypy/nodes.py | 25 ++--------- mypy/semanal.py | 97 ++++++++++++++++++++++++++++++++++++++++++- mypy/strconv.py | 3 ++ mypy/treetransform.py | 5 ++- mypy/typeanal.py | 2 +- 7 files changed, 116 insertions(+), 26 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 31cb2371c1be..76b229b8e405 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -20,7 +20,7 @@ BytesExpr, UnicodeExpr, FloatExpr, OpExpr, UnaryExpr, CastExpr, RevealTypeExpr, SuperExpr, TypeApplication, DictExpr, SliceExpr, FuncExpr, TempNode, SymbolTableNode, Context, ListComprehension, ConditionalExpr, GeneratorExpr, - Decorator, SetExpr, TypeVarExpr, PrintStmt, + Decorator, SetExpr, TypeVarExpr, NewTypeExpr, PrintStmt, LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr, YieldFromExpr, NamedTupleExpr, SetComprehension, DictionaryComprehension, ComplexExpr, EllipsisExpr, TypeAliasExpr, @@ -1979,6 +1979,9 @@ def visit_type_var_expr(self, e: TypeVarExpr) -> Type: # TODO: Perhaps return a special type used for type variables only? return AnyType() + def visit_newtype_expr(self, e: NewTypeExpr) -> Type: + return AnyType() + def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type: # TODO: Perhaps return a type object type? return AnyType() diff --git a/mypy/fixup.py b/mypy/fixup.py index 929da2426508..80fdb71568ab 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -4,7 +4,7 @@ from mypy.nodes import (MypyFile, SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, FuncDef, OverloadedFuncDef, Decorator, Var, - TypeVarExpr, ClassDef, + TypeVarExpr, NewTypeExpr, ClassDef, LDEF, MDEF, GDEF, MODULE_REF) from mypy.types import (CallableType, EllipsisType, Instance, Overloaded, TupleType, TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor, @@ -123,6 +123,9 @@ def visit_type_var_expr(self, tv: TypeVarExpr) -> None: value.accept(self.type_fixer) tv.upper_bound.accept(self.type_fixer) + def visit_newtype_expr(self, nt: NewTypeExpr) -> None: + raise NotImplemented() + def visit_var(self, v: Var) -> None: if self.current_info is not None: v.info = self.current_info diff --git a/mypy/nodes.py b/mypy/nodes.py index 2c4c2f6a892d..02763a7d9829 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1710,34 +1710,17 @@ def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit__promote_expr(self) -class NewTypeExpr(SymbolNode, Expression): +class NewTypeExpr(Expression): """NewType expression NewType(...).""" - name = '' - fullname = '' - value = None # type: mypy.types.Type + info = None # type: TypeInfo - def __init__(self, name: str, fullname: str, value: 'mypy.type.Type') -> None: - self.name = name - self.fullname = fullname - self.value = value + def __init__(self, info: 'TypeInfo') -> None: + self.info = info def accept(self, visitor: NodeVisitor[T]) -> T: return visitor.visit_newtype_expr(self) - def serialize(self) -> JsonDict: - return {'.class': 'NewTypeExpr', - 'name': self.name, - 'fullname': self.fullname, - 'value': self.value.serialize()} - - @classmethod - def deserialize(cls, data, JsonDict) -> 'TypeVarExpr': - assert data['.class'] == 'NewTypeExpr' - return NewTypeExpr(data['name'], - data['fullname'], - mypy.types.Type.deserialize(data['value'])) - class AwaitExpr(Node): """Await expression (await ...).""" diff --git a/mypy/semanal.py b/mypy/semanal.py index 4401183dbdf5..f020afe66788 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -57,7 +57,7 @@ GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, - FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, + FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, NewTypeExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases, YieldFromExpr, NamedTupleExpr, NonlocalDecl, @@ -1082,6 +1082,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: for lvalue in s.lvalues: self.store_declared_types(lvalue, s.type) self.check_and_set_up_type_alias(s) + self.process_newtype_declaration(s) self.process_typevar_declaration(s) self.process_namedtuple_definition(s) @@ -1288,6 +1289,93 @@ def store_declared_types(self, lvalue: Node, typ: Type) -> None: # This has been flagged elsewhere as an error, so just ignore here. pass + def process_newtype_declaration(self, s: AssignmentStmt) -> None: + """Check if s declares a NewType; if yes, store it in symbol table.""" + call = self.get_newtype_declaration(s) + if not call: + return + + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a newtype variable", s) + else: + self.fail("Cannot redefine '%s' as a newtype" % name, s) + return + + if not self.check_newtype_name(call, name, s) or call.arg_kinds[1] != ARG_POS: + return + + value = self.analyze_types(call.args[1:])[0] + # TODO: determine if value can actually ever be None + assert value is not None + assert isinstance(value, Instance) + + # Looks like a valid newtype! Create the corresponding class def... + newtype_class_info = self.build_newtype_typeinfo(name, value) + + # ...and add it to the symbol table. + node = self.lookup(name, s) + node.kind = GDEF # TODO: locally defined newtype + call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) + node.node = newtype_class_info + + def build_newtype_typeinfo(self, name: str, value: Instance) -> TypeInfo: + symbols = SymbolTable() + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + info = TypeInfo(symbols, class_def) + info.mro = [info] + value.type.mro + info.bases = [value] + + # Add __init__ method + args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), + self.make_argument('item', value)] + signature = CallableType( + arg_types = [cast(Type, None), value], + arg_kinds = [arg.kind for arg in args], + arg_names = ['self', 'item'], + ret_type = value, + fallback = self.named_type('__builtins__.function'), + name = info.name()) + init_func = FuncDef('__init__', args, Block([]), typ=signature) + init_func.info = info + symbols['__init__'] = SymbolTableNode(MDEF, init_func) + + return info + + def check_newtype_name(self, call: CallExpr, name: str, context: Context) -> bool: + args = call.args + if len(args) != 2: + direction = "few" if len(args) < 2 else "many" + self.fail("Too {} arguments for NewType()".format(direction), context) + return False + if (not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)) + or not call.arg_kinds[0] == ARG_POS): + self.fail("NewType(...) expects a string literal as first argument", context) + return False + if cast(StrExpr, call.args[0]).value != name: + self.fail("First argument to NewType() does not match variable name", context) + return False + return True + + def get_newtype_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the Newtype() call statement if `s` is a newtype declaration + or None otherwise.""" + # TODO: determine if this and get_typevar_declaration should be refactored + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.NewType': + return None + return call + def process_typevar_declaration(self, s: AssignmentStmt) -> None: """Check if s declares a TypeVar; it yes, store it in symbol table.""" call = self.get_typevar_declaration(s) @@ -2080,6 +2168,9 @@ def visit_backquote_expr(self, expr: BackquoteExpr) -> None: def visit__promote_expr(self, expr: PromoteExpr) -> None: expr.type = self.anal_type(expr.type) + def visit_newtype_expr(self, expr: NewTypeExpr) -> None: + raise NotImplemented() + def visit_yield_expr(self, expr: YieldExpr) -> None: if not self.is_func_scope(): self.fail("'yield' outside function", expr, True, blocker=True) @@ -2524,6 +2615,10 @@ def visit_if_stmt(self, s: IfStmt) -> None: def visit_try_stmt(self, s: TryStmt) -> None: self.sem.analyze_try_stmt(s, self, add_global=True) + def visit_newtype_expr(self, e: NewTypeExpr) -> None: + # TODO: This function can probably be deleted + raise NotImplemented() + def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) diff --git a/mypy/strconv.py b/mypy/strconv.py index cb48f8ec045c..c2461ca16518 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -433,6 +433,9 @@ def visit_namedtuple_expr(self, o): def visit__promote_expr(self, o): return 'PromoteExpr:{}({})'.format(o.line, o.type) + def visit_newtype_expr(self, o): + return 'NewTypeExpr:{}({}, {})'.format(o.line, o.fullname(), self.dump([o.value], o)) + def visit_func_expr(self, o): a = self.func_helper(o) return self.dump(a, o) diff --git a/mypy/treetransform.py b/mypy/treetransform.py index f05232586b14..06ea5cbed629 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -15,7 +15,7 @@ ConditionalExpr, DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr, FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr, UnaryExpr, FuncExpr, TypeApplication, PrintStmt, - SymbolTable, RefExpr, TypeVarExpr, PromoteExpr, + SymbolTable, RefExpr, TypeVarExpr, NewTypeExpr, PromoteExpr, ComparisonExpr, TempNode, StarExpr, YieldFromExpr, NamedTupleExpr, NonlocalDecl, SetComprehension, DictionaryComprehension, ComplexExpr, TypeAliasExpr, EllipsisExpr, @@ -453,6 +453,9 @@ def visit_type_var_expr(self, node: TypeVarExpr) -> Node: def visit_type_alias_expr(self, node: TypeAliasExpr) -> TypeAliasExpr: return TypeAliasExpr(node.type) + def visit_newtype_expr(self, node: NewTypeExpr) -> NewTypeExpr: + return NewTypeExpr(node.info) + def visit_namedtuple_expr(self, node: NamedTupleExpr) -> Node: return NamedTupleExpr(node.info) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0493d3a15f6b..19c7130bcd26 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,7 @@ from mypy.nodes import ( BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr + IndexExpr, RefExpr, NewTypeExpr ) from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError From a9c8205d549c6369c4501e8053b73d6a3780e0ea Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 25 Jul 2016 17:15:40 -0700 Subject: [PATCH 03/26] Add basic tests for NewType --- mypy/test/testcheck.py | 1 + test-data/unit/check-newtype.test | 38 +++++++++++++++++++++++++++++++ test-data/unit/lib-stub/typing.py | 6 +++++ 3 files changed, 45 insertions(+) create mode 100644 test-data/unit/check-newtype.test diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 1bc7f83834fb..9f8a26837bff 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -64,6 +64,7 @@ 'check-fastparse.test', 'check-warnings.test', 'check-async-await.test', + 'check-newtype.test', ] diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test new file mode 100644 index 000000000000..4320cd215fc5 --- /dev/null +++ b/test-data/unit/check-newtype.test @@ -0,0 +1,38 @@ +-- Checks NewType(...) + +[case testNewTypePEP484Example1] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +UserId('user') # E: Argument 1 to "UserId" has incompatible type "str"; expected "int" +name_by_id(42) # E: Argument 1 to "name_by_id" has incompatible type "int"; expected "UserId" +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +reveal_type(id) # E: Revealed type is '__main__.UserId' +reveal_type(num) # E: Revealed type is 'builtins.int' +[out] + +[case testNewTypePEP484Example2] +from typing import NewType + +class PacketId: + def __init__(self, major: int, minor: int) -> None: + self._major = major + self._minor = minor + +TcpPacketId = NewType('TcpPacketId', PacketId) + +packet = PacketId(100, 100) +tcp_packet = TcpPacketId(packet) +tcp_packet = TcpPacketId(127, 0) + +[out] +main:12: error: Too many arguments for "TcpPacketId" +main:12: error: Argument 1 to "TcpPacketId" has incompatible type "int"; expected "PacketId" diff --git a/test-data/unit/lib-stub/typing.py b/test-data/unit/lib-stub/typing.py index 3e539f1f5e02..77fb3a751c5a 100644 --- a/test-data/unit/lib-stub/typing.py +++ b/test-data/unit/lib-stub/typing.py @@ -73,3 +73,9 @@ def __anext__(self) -> Awaitable[T]: pass class Sequence(Generic[T]): @abstractmethod def __getitem__(self, n: Any) -> T: pass + +def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: + def new_type(x): + return x + return new_type + From e8b1cd64f03df0c9368fca7cef8355592575821b Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 25 Jul 2016 17:25:12 -0700 Subject: [PATCH 04/26] Fix lint error --- mypy/semanal.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f020afe66788..cf45c927f553 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1333,12 +1333,12 @@ def build_newtype_typeinfo(self, name: str, value: Instance) -> TypeInfo: args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), self.make_argument('item', value)] signature = CallableType( - arg_types = [cast(Type, None), value], - arg_kinds = [arg.kind for arg in args], - arg_names = ['self', 'item'], - ret_type = value, - fallback = self.named_type('__builtins__.function'), - name = info.name()) + arg_types = [cast(Type, None), value], + arg_kinds = [arg.kind for arg in args], + arg_names = ['self', 'item'], + ret_type = value, + fallback = self.named_type('__builtins__.function'), + name = info.name()) init_func = FuncDef('__init__', args, Block([]), typ=signature) init_func.info = info symbols['__init__'] = SymbolTableNode(MDEF, init_func) From 582c651cd90ae329e4b53c69a8f40438cc50a052 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 13:47:53 -0700 Subject: [PATCH 05/26] Add is_newtype flag to nodes to mirror namedtuple --- mypy/nodes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy/nodes.py b/mypy/nodes.py index 02763a7d9829..291e07216542 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1810,6 +1810,9 @@ class is generic then it will be a type constructor of higher kind. # Is this a named tuple type? is_named_tuple = False + # Is this a newtype type? + is_newtype = False + # Is this a dummy from deserialization? is_dummy = False @@ -1986,6 +1989,7 @@ def serialize(self) -> Union[str, JsonDict]: '_promote': None if self._promote is None else self._promote.serialize(), 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), 'is_named_tuple': self.is_named_tuple, + 'is_newtype': self.is_newtype } return data @@ -2008,6 +2012,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': ti.tuple_type = (None if data['tuple_type'] is None else mypy.types.TupleType.deserialize(data['tuple_type'])) ti.is_named_tuple = data['is_named_tuple'] + ti.is_newtype = data['is_newtype'] return ti From eea40bf35ea3c7029a923765f54f64f9e138a882 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 13:56:22 -0700 Subject: [PATCH 06/26] Add some incomplete tests for NewType --- test-data/unit/check-newtype.test | 98 +++++++++++++++++++++++++++++++ test-data/unit/fixtures/tuple.py | 13 +++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 4320cd215fc5..48812fa1754b 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -36,3 +36,101 @@ tcp_packet = TcpPacketId(127, 0) [out] main:12: error: Too many arguments for "TcpPacketId" main:12: error: Argument 1 to "TcpPacketId" has incompatible type "int"; expected "PacketId" + +[case testNewTypeBadInitialization] +from typing import NewType + +a = NewType('b', int) +b = NewType('b', 3) +c = NewType(2, int) +foo = "d" +d = NewType(foo, int) +e = NewType(name='e', tp=int) +f = NewType('f', tp=int) +[out] +main:3: error: Argument 1 to NewType(...) does not match variable name +main:4: error: Argument 2 to NewType(...) must be a valid type +main:5: error: Argument 1 to NewType(...) must be a string literal +main:7: error: Argument 1 to NewType(...) must be a string literal +main:8: error: Argument 1 to NewType(...) must be a positional string literal +main:8: error: Argument 2 to NewType(...) must be a positional type +main:9: error: Argument 2 to NewType(...) must be a positional type + +[case testNewTypeWithCasts] +from typing import NewType, cast +UserId = NewType('UserId', int) +foo = UserId(3) +foo = cast(UserId, 3) +foo = cast(UserId, "foo") +foo = cast(UserId, UserId(4)) +[out] + +[case testNewTypeWithCompositeTypes] +from typing import NewType, Tuple, List +TwoTuple = NewType('TwoTuple', Tuple[int, str]) +a = TwoTuple((3, "a")) +b = TwoTuple(("a", 3)) # E: Argument 1 to "TwoTuple" has incompatible type "Tuple[str, int]"; expected "Tuple[int, str]" + +UserId = NewType('UserId', int) +IdList = NewType('IdList', List[UserId]) + +bad1 = IdList([1]) # E: List item 0 has incompatible type "int" + +foo = IdList([]) +foo.append(3) # E: Argument 1 to "append" of "list" has incompatible type "int"; expected "UserId" +foo.append(UserId(3)) +foo.extend([UserId(1), UserId(2), UserId(3)]) +foo.extend(IdList([UserId(1), UserId(2), UserId(3)])) +bar = IdList([UserId(2)]) + +#baz = foo + bar +reveal_type(foo) # E: Revealed type is '__main__.IdList' +reveal_type(bar) # E: Revealed type is '__main__.IdList' +#reveal_type(baz) # TODO: Revealed type is 'builtins.list' + +[builtins fixtures/tuple.py] +[out] + +[case testNewTypeWithUnions] +from typing import NewType, Union +Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) + +[out] + +[case testNewTypeWithGenerics] +from typing import TypeVar, Generic, NewType, Any + +T = TypeVar('T') + +class Base(Generic[T]): + def __init__(self, item: T) -> None: + self.item = item + +Derived1 = NewType('Derived1', Base[str]) +Derived2 = NewType('Derived2', Base) # Implicit 'Any' +Derived3 = NewType('Derived3', Base[Any]) # Explicit 'Any' + +Derived1(Base(1)) # E: Argument 1 to "Base" has incompatible type "int"; expected "str" +Derived1(Base('a')) +Derived2(Base(1)) +Derived2(Base('a')) +Derived3(Base(1)) +Derived3(Base('a')) +[out] + +[case testNewTypeInMultipleFiles] +import a +import b +list1 = [a.UserId(1), a.UserId(2)] +list1.append(b.UserId(3)) # E: Argument 1 to "append" of "list" has incompatible type "b.UserId"; expected "a.UserId" + +[file a.py] +from typing import NewType +UserId = NewType('UserId', int) + +[file b.py] +from typing import NewType +UserId = NewType('UserId', int) + +[builtins fixtures/list.py] +[out] diff --git a/test-data/unit/fixtures/tuple.py b/test-data/unit/fixtures/tuple.py index 76c109127364..75bffe445b7b 100644 --- a/test-data/unit/fixtures/tuple.py +++ b/test-data/unit/fixtures/tuple.py @@ -1,6 +1,6 @@ # Builtins stub used in tuple-related test cases. -from typing import Iterable, TypeVar, Generic, Sequence +from typing import Iterable, Iterator, TypeVar, Generic, Sequence, overload Tco = TypeVar('Tco', covariant=True) @@ -24,3 +24,14 @@ class str: pass # For convenience def sum(iterable: Iterable[T], start: T = None) -> T: pass True = bool() + +class list(Iterable[T], Generic[T]): + @overload + def __init__(self) -> None: pass + @overload + def __init__(self, x: Iterable[T]) -> None: pass + def __iter__(self) -> Iterator[T]: pass + def __mul__(self, x: int) -> list[T]: pass + def __getitem__(self, x: int) -> T: pass + def append(self, x: T) -> None: pass + def extend(self, x: Iterable[T]) -> None: pass From 2f604a09b4b626bf4562fbb8d1f45830a48f2548 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 13:56:55 -0700 Subject: [PATCH 07/26] Add refinements to semantic analysis for NewType --- mypy/semanal.py | 79 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index cf45c927f553..310f066b0c3f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1304,16 +1304,22 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: self.fail("Cannot redefine '%s' as a newtype" % name, s) return - if not self.check_newtype_name(call, name, s) or call.arg_kinds[1] != ARG_POS: + underlying_type = self.check_newtype_args(call, name, s) + if underlying_type is None: return - value = self.analyze_types(call.args[1:])[0] - # TODO: determine if value can actually ever be None - assert value is not None - assert isinstance(value, Instance) + # Check if class is subtypeable + if isinstance(underlying_type, TupleType): + base_type = underlying_type.fallback + elif isinstance(underlying_type, Instance): + base_type = underlying_type + else: + message = "Argument 2 to NewType(...) must be subclassable (got {})" + self.fail_blocker(message.format(underlying_type), s) + return - # Looks like a valid newtype! Create the corresponding class def... - newtype_class_info = self.build_newtype_typeinfo(name, value) + # Create the corresponding class def... + newtype_class_info = self.build_newtype_typeinfo(name, underlying_type, base_type) # ...and add it to the symbol table. node = self.lookup(name, s) @@ -1321,44 +1327,63 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) node.node = newtype_class_info - def build_newtype_typeinfo(self, name: str, value: Instance) -> TypeInfo: - symbols = SymbolTable() + def build_newtype_typeinfo(self, name: str, underlying_type: Type, base_type: Instance) -> TypeInfo: class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) + + symbols = SymbolTable() info = TypeInfo(symbols, class_def) - info.mro = [info] + value.type.mro - info.bases = [value] + info.mro = [info] + base_type.type.mro + info.bases = [base_type] + info.is_newtype = True # Add __init__ method args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), - self.make_argument('item', value)] + self.make_argument('item', underlying_type)] signature = CallableType( - arg_types = [cast(Type, None), value], + arg_types = [cast(Type, None), underlying_type], arg_kinds = [arg.kind for arg in args], arg_names = ['self', 'item'], - ret_type = value, + ret_type = underlying_type, fallback = self.named_type('__builtins__.function'), - name = info.name()) + name = name) init_func = FuncDef('__init__', args, Block([]), typ=signature) init_func.info = info symbols['__init__'] = SymbolTableNode(MDEF, init_func) return info - def check_newtype_name(self, call: CallExpr, name: str, context: Context) -> bool: + def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Type: + has_failed = False args = call.args if len(args) != 2: - direction = "few" if len(args) < 2 else "many" - self.fail("Too {} arguments for NewType()".format(direction), context) - return False - if (not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)) - or not call.arg_kinds[0] == ARG_POS): - self.fail("NewType(...) expects a string literal as first argument", context) - return False - if cast(StrExpr, call.args[0]).value != name: - self.fail("First argument to NewType() does not match variable name", context) - return False - return True + self.fail("NewType(...) expects exactly two arguments", context) + return None + + # Check first argument + string_types = (StrExpr, BytesExpr, UnicodeExpr) + if not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)): + self.fail("Argument 1 to NewType(...) must be a string literal", context) + has_failed = True + elif call.arg_kinds[0] != ARG_POS: + self.fail("Argument 1 to NewType(...) must be a positional string literal", context) + has_failed = True + elif cast(StrExpr, call.args[0]).value != name: + self.fail("Argument 1 to NewType(...) does not match variable name", context) + has_failed = True + + # Check second argument + try: + value = self.anal_type(expr_to_unanalyzed_type(call.args[1])) + except TypeTranslationError: + self.fail_blocker("Argument 2 to NewType(...) must be a valid type", context) + return None + + if call.arg_kinds[1] != ARG_POS: + self.fail("Argument 2 to NewType(...) must be a positional type", context) + has_failed = True + + return None if has_failed else value def get_newtype_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: """Returns the Newtype() call statement if `s` is a newtype declaration From 868f90cf0b941039d0e85e6cf5ab585ffc3e4203 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 14:22:50 -0700 Subject: [PATCH 08/26] Fix error in tests --- test-data/unit/check-newtype.test | 4 ++-- test-data/unit/fixtures/tuple.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 48812fa1754b..3a7b224742d2 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -83,10 +83,10 @@ foo.extend([UserId(1), UserId(2), UserId(3)]) foo.extend(IdList([UserId(1), UserId(2), UserId(3)])) bar = IdList([UserId(2)]) -#baz = foo + bar +baz = foo + bar reveal_type(foo) # E: Revealed type is '__main__.IdList' reveal_type(bar) # E: Revealed type is '__main__.IdList' -#reveal_type(baz) # TODO: Revealed type is 'builtins.list' +reveal_type(baz) # E: Revealed type is 'builtins.list[__main__.UserId*]' [builtins fixtures/tuple.py] [out] diff --git a/test-data/unit/fixtures/tuple.py b/test-data/unit/fixtures/tuple.py index 75bffe445b7b..2d0cab5b6f02 100644 --- a/test-data/unit/fixtures/tuple.py +++ b/test-data/unit/fixtures/tuple.py @@ -32,6 +32,7 @@ def __init__(self) -> None: pass def __init__(self, x: Iterable[T]) -> None: pass def __iter__(self) -> Iterator[T]: pass def __mul__(self, x: int) -> list[T]: pass + def __add__(self, x: list[T]) -> list[T]: pass def __getitem__(self, x: int) -> T: pass def append(self, x: T) -> None: pass def extend(self, x: Iterable[T]) -> None: pass From c31e01da2b4cf2e956f7b9dab58f47eaca00ea76 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 16:19:02 -0700 Subject: [PATCH 09/26] Add more tests --- test-data/unit/check-newtype.test | 83 ++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 3a7b224742d2..15f51e47e03d 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -71,6 +71,9 @@ TwoTuple = NewType('TwoTuple', Tuple[int, str]) a = TwoTuple((3, "a")) b = TwoTuple(("a", 3)) # E: Argument 1 to "TwoTuple" has incompatible type "Tuple[str, int]"; expected "Tuple[int, str]" +reveal_type(a[0]) # E: Revealed type is 'builtins.int' +reveal_type(a[1]) # E: Revealed type is 'builtins.str' + UserId = NewType('UserId', int) IdList = NewType('IdList', List[UserId]) @@ -94,7 +97,6 @@ reveal_type(baz) # E: Revealed type is 'builtins.list[__main__.UserId*]' [case testNewTypeWithUnions] from typing import NewType, Union Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) - [out] [case testNewTypeWithGenerics] @@ -106,6 +108,9 @@ class Base(Generic[T]): def __init__(self, item: T) -> None: self.item = item + def getter(self) -> T: + return self.item + Derived1 = NewType('Derived1', Base[str]) Derived2 = NewType('Derived2', Base) # Implicit 'Any' Derived3 = NewType('Derived3', Base[Any]) # Explicit 'Any' @@ -116,6 +121,9 @@ Derived2(Base(1)) Derived2(Base('a')) Derived3(Base(1)) Derived3(Base('a')) + +reveal_type(Derived1(Base('a')).getter()) # E: Revealed type is 'builtins.str*' +reveal_type(Derived3(Base('a')).getter()) # E: Revealed type is 'Any' [out] [case testNewTypeInMultipleFiles] @@ -134,3 +142,76 @@ UserId = NewType('UserId', int) [builtins fixtures/list.py] [out] + +[case testNewTypeWithTypeAliases] +from typing import NewType +Foo = int +Bar = NewType('Bar', Foo) + +def func1(x: Foo) -> Bar: + return Bar(x) + +def func2(x: int) -> Bar: + return Bar(x) + +[out] + +[case testNewTypeWithTypeType] +from typing import NewType, Type +Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got Type[builtins.int]) +a = Foo(type(3)) +[builtins fixtures/args.py] +[out] + +[case testNewTypeWithTypeVars] +from typing import NewType, TypeVar, List +T = TypeVar('T') +A = NewType('A', T) +B = NewType('B', List[T]) +[builtins fixtures/list.py] +[out] +main:3: error: Invalid type "__main__.T" +main:3: error: Argument 2 to NewType(...) must be subclassable (got T?) +main:4: error: Invalid type "__main__.T" + +[case testNewTypeRedefiningVariables] +from typing import NewType + +a = 3 +a = NewType('a', int) + +b = NewType('b', int) +b = NewType('b', float) + +c = NewType('c', str) # type: str +[out] +main:4: error: Cannot redefine 'a' as a NewType +main:7: error: Invalid assignment target +main:7: error: Cannot redefine 'b' as a NewType +main:9: error: Cannot declare the type of a NewType declaration + +[case testNewTypeInLocalScope] +from typing import NewType +A = NewType('A', int) +a = A(3) + +def func() -> None: + A = NewType('A', str) + B = NewType('B', str) + + a = A(3) + a = A('xyz') + b = B('xyz') + +class MyClass: + C = NewType('C', float) + + def foo(self) -> 'MyClass.C': + return MyClass.C(3.2) + +b = A(3) +c = MyClass.C(3.5) +[out] +main: note: In function "func": +main:9: error: Argument 1 to "A" has incompatible type "int"; expected "str" + From ccc1032989e6055b6c6ba1faab8d91c28cff644d Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 16:19:25 -0700 Subject: [PATCH 10/26] Modify NewType handling to fix bugs --- mypy/nodes.py | 4 ++-- mypy/semanal.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 291e07216542..769e42e88534 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1713,9 +1713,9 @@ def accept(self, visitor: NodeVisitor[T]) -> T: class NewTypeExpr(Expression): """NewType expression NewType(...).""" - info = None # type: TypeInfo + info = None # type: Optional[TypeInfo] - def __init__(self, info: 'TypeInfo') -> None: + def __init__(self, info: Optional['TypeInfo']) -> None: self.info = info def accept(self, visitor: NodeVisitor[T]) -> T: diff --git a/mypy/semanal.py b/mypy/semanal.py index 310f066b0c3f..78908f7166d5 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -71,7 +71,7 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, Void, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, TypeType, Void, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType ) from mypy.nodes import function_type, implicit_module_attrs @@ -1294,14 +1294,15 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: call = self.get_newtype_declaration(s) if not call: return + call.analyzed = NewTypeExpr(None).set_line(call.line) lvalue = cast(NameExpr, s.lvalues[0]) name = lvalue.name if not lvalue.is_def: if s.type: - self.fail("Cannot declare the type of a newtype variable", s) + self.fail("Cannot declare the type of a NewType declaration", s) else: - self.fail("Cannot redefine '%s' as a newtype" % name, s) + self.fail("Cannot redefine '%s' as a NewType" % name, s) return underlying_type = self.check_newtype_args(call, name, s) @@ -1315,7 +1316,7 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: base_type = underlying_type else: message = "Argument 2 to NewType(...) must be subclassable (got {})" - self.fail_blocker(message.format(underlying_type), s) + self.fail(message.format(underlying_type), s) return # Create the corresponding class def... @@ -1327,7 +1328,10 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) node.node = newtype_class_info - def build_newtype_typeinfo(self, name: str, underlying_type: Type, base_type: Instance) -> TypeInfo: + def build_newtype_typeinfo(self, + name: str, + underlying_type: Type, + base_type: Instance) -> TypeInfo: class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) @@ -1336,6 +1340,8 @@ def build_newtype_typeinfo(self, name: str, underlying_type: Type, base_type: In info.mro = [info] + base_type.type.mro info.bases = [base_type] info.is_newtype = True + if isinstance(underlying_type, TupleType): + info.tuple_type = underlying_type # Add __init__ method args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), @@ -1361,7 +1367,6 @@ def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Typ return None # Check first argument - string_types = (StrExpr, BytesExpr, UnicodeExpr) if not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)): self.fail("Argument 1 to NewType(...) must be a string literal", context) has_failed = True @@ -1376,7 +1381,7 @@ def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Typ try: value = self.anal_type(expr_to_unanalyzed_type(call.args[1])) except TypeTranslationError: - self.fail_blocker("Argument 2 to NewType(...) must be a valid type", context) + self.fail("Argument 2 to NewType(...) must be a valid type", context) return None if call.arg_kinds[1] != ARG_POS: From b022eb3c9adc3f17cee3fbe968c6f19e0e43a79d Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 16:52:03 -0700 Subject: [PATCH 11/26] Add tests to check for bad subtyping --- test-data/unit/check-newtype.test | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 15f51e47e03d..4d627d520026 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -215,3 +215,55 @@ c = MyClass.C(3.5) main: note: In function "func": main:9: error: Argument 1 to "A" has incompatible type "int"; expected "str" +[case testNewTypeTestSubclassingFails] +from typing import NewType +class A: pass +B = NewType('B', A) +class C(B): pass # E: Cannot subclass NewType +[out] + +[case testNewTypeOfANewTypeFails] +from typing import NewType +A = NewType('A', int) +B = NewType('B', A) # E: Argument 2 to NewType(...) cannot be another NewType +C = A +D = C +E = NewType('E', D) # E: Argument 2 to NewType(...) cannot be another NewType +[out] + +[case testIncrementalWithNewType] +import m + +[file m.py] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +[file m.py.next] +from typing import NewType + +UserId = NewType('UserId', int) + +def name_by_id(user_id: UserId) -> str: + return "foo" + +name_by_id(UserId(42)) + +id = UserId(5) +num = id + 1 + +reveal_type(id) +reveal_type(num) +[stale m] +[out] +main:1: note: In module imported here: +tmp/m.py:13: error: Revealed type is 'm.UserId' +tmp/m.py:14: error: Revealed type is 'builtins.int' From 06a4e9e9575e8b8efc980fc046fcfaf26fa1a219 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 16:52:40 -0700 Subject: [PATCH 12/26] Modify semanal to catch bad subtyping with NewType --- mypy/semanal.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 78908f7166d5..605c15ef7601 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -768,6 +768,8 @@ def analyze_base_classes(self, defn: ClassDef) -> None: defn.info.tuple_type = base base_types.append(base.fallback) elif isinstance(base, Instance): + if base.type.is_newtype: + self.fail("Cannot subclass NewType", defn) base_types.append(base) elif isinstance(base, AnyType): defn.info.fallback_to_any = True @@ -1387,6 +1389,8 @@ def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Typ if call.arg_kinds[1] != ARG_POS: self.fail("Argument 2 to NewType(...) must be a positional type", context) has_failed = True + elif isinstance(value, Instance) and value.type.is_newtype: + self.fail("Argument 2 to NewType(...) cannot be another NewType", context) return None if has_failed else value From 385a4b69a43c4cba44dabe50f8bfc97e9bdb3dc9 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 17:03:29 -0700 Subject: [PATCH 13/26] Add test for namedtuples --- test-data/unit/check-newtype.test | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 4d627d520026..74aed33bc6fe 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -231,6 +231,36 @@ D = C E = NewType('E', D) # E: Argument 2 to NewType(...) cannot be another NewType [out] +[case testNewTypeWithNamedTuple] +from collections import namedtuple +from typing import NewType, NamedTuple + +Vector1 = namedtuple('Vector1', ['x', 'y', 'z']) +Point1 = NewType('Point1', Vector1) +p1 = Point1(Vector1(1, 2, 3)) +reveal_type(p1.x) # E: Revealed type is 'Any' +reveal_type(p1.y) # E: Revealed type is 'Any' +reveal_type(p1.z) # E: Revealed type is 'Any' + +Vector2 = NamedTuple('Vector2', [('x', int), ('y', int), ('z', int)]) +Point2 = NewType('Point2', Vector2) +p2 = Point2(Vector2(1, 2, 3)) +reveal_type(p2.x) # E: Revealed type is 'builtins.int' +reveal_type(p2.y) # E: Revealed type is 'builtins.int' +reveal_type(p2.z) # E: Revealed type is 'builtins.int' + +class Vector3: + def __init__(self, x: int, y: int, z: int) -> None: + self.x = x + self.y = y + self.z = z +Point3 = NewType('Point3', Vector3) +p3 = Point3(Vector3(1, 3, 3)) +reveal_type(p3.x) # E: Revealed type is 'builtins.int' +reveal_type(p3.y) # E: Revealed type is 'builtins.int' +reveal_type(p3.z) # E: Revealed type is 'builtins.int' +[out] + [case testIncrementalWithNewType] import m From 609746c2a1709ea031b467c7bcc9fcff55de0f7e Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Tue, 26 Jul 2016 17:18:37 -0700 Subject: [PATCH 14/26] Add info about NewType to documentation --- docs/source/kinds_of_types.rst | 109 +++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 88007da332ff..99efcf9a8af0 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -425,6 +425,115 @@ A type alias does not create a new type. It's just a shorthand notation for another type -- it's equivalent to the target type. Type aliases can be imported from modules like any names. +.. _newtypes: + +NewTypes +******** + +(Freely after `PEP 484 +`_.) + +There are also situations where a programmer might want to avoid logical errors by +creating simple classes. For example: + +.. code-block:: python + + class UserId(int): + pass + + get_by_user_id(user_id: UserId): + ... + +However, this approach introduces some runtime overhead. To avoid this, the typing +module provides a helper function ``NewType`` that creates simple unique types with +almost zero runtime overhead. Mypy will treat the statement +``Derived = NewType('Derived', Base)`` as being roughly equivalent to the following +definition: + +.. code-block:: python + + class Derived(Base): + def __init__(self, _x: Base) -> None: + ... + +However, at runtime, ``NewType('Derived', Base)`` will return a dummy function that +simply returns its argument. + +Mypy will require explicit casts from ``int`` where ``UserId`` is expected, while +implicitly casting from ``UserId`` where ``int`` is expected. Examples: + +.. code-block:: python + + from typing import NewType + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + UserId('user') # Fails type check + + name_by_id(42) # Fails type check + name_by_id(UserId(42)) # OK + + num = UserId(5) + 1 # type: int + +``NewType`` accepts only one argument that must be a properly subclassable class, i.e., +not a type construct like ``Union``, etc. The function returned by ``NewType`` accepts +only one argument; this is equivalent to supporting only one constructor accepting an +instance of the base class (see above). Example: + +.. code-block:: python + + from typing import NewType + + class PacketId: + def __init__(self, major: int, minor: int) -> None: + self._major = major + self._minor = minor + + TcpPacketId = NewType('TcpPacketId', PacketId) + + packet = PacketId(100, 100) + tcp_packet = TcpPacketId(packet) # OK + + tcp_packet = TcpPacketId(127, 0) # Fails in type checker and at runtime + +Both ``isinstance`` and ``issubclass``, as well as subclassing will fail for +``NewType('Derived', Base)`` since function objects don't support these operations. + +.. note:: + + Note that unlike type aliases, ``NewType`` will create an entirely new and + unique type when used. The intended purpose of ``NewType`` is to help you + detect cases where you accidentally mixed together the old base type and the + new derived type. + + For example, the following will successfully typecheck when using type + aliases: + + .. code-block:: python + + UserId = int + + def name_by_id(user_id: UserId) -> str: + ... + + name_by_id(3) # ints and UserId are synonymous + + But a similar example using ``NewType`` will not typecheck: + + .. code-block:: python + + from typing import NewType + + UserId = NewType('UserId', int) + + def name_by_id(user_id: UserId) -> str: + ... + + name_by_id(3) # int is not the same as UserId + .. _named-tuples: Named tuples From 7967bcffe3118429429b3dc255e42141ed67d0a4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 08:16:11 -0700 Subject: [PATCH 15/26] Remove unneeded code and imports --- mypy/semanal.py | 7 ------- mypy/typeanal.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 605c15ef7601..66649b5a10f4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2202,9 +2202,6 @@ def visit_backquote_expr(self, expr: BackquoteExpr) -> None: def visit__promote_expr(self, expr: PromoteExpr) -> None: expr.type = self.anal_type(expr.type) - def visit_newtype_expr(self, expr: NewTypeExpr) -> None: - raise NotImplemented() - def visit_yield_expr(self, expr: YieldExpr) -> None: if not self.is_func_scope(): self.fail("'yield' outside function", expr, True, blocker=True) @@ -2649,10 +2646,6 @@ def visit_if_stmt(self, s: IfStmt) -> None: def visit_try_stmt(self, s: TryStmt) -> None: self.sem.analyze_try_stmt(s, self, add_global=True) - def visit_newtype_expr(self, e: NewTypeExpr) -> None: - # TODO: This function can probably be deleted - raise NotImplemented() - def analyze_lvalue(self, lvalue: Node, explicit_type: bool = False) -> None: self.sem.analyze_lvalue(lvalue, add_global=True, explicit_type=explicit_type) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 19c7130bcd26..0493d3a15f6b 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,7 +10,7 @@ from mypy.nodes import ( BOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, TypeVarExpr, Var, Node, - IndexExpr, RefExpr, NewTypeExpr + IndexExpr, RefExpr ) from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError From 0daf4863ec2173b27918131b24f0bc47df9f36e4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 09:01:41 -0700 Subject: [PATCH 16/26] Rearrange newtype tests to be more logical --- test-data/unit/check-newtype.test | 245 +++++++++++++++--------------- 1 file changed, 126 insertions(+), 119 deletions(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 74aed33bc6fe..94de48f3455b 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -1,5 +1,7 @@ -- Checks NewType(...) +-- Checks for basic functionality + [case testNewTypePEP484Example1] from typing import NewType @@ -37,34 +39,6 @@ tcp_packet = TcpPacketId(127, 0) main:12: error: Too many arguments for "TcpPacketId" main:12: error: Argument 1 to "TcpPacketId" has incompatible type "int"; expected "PacketId" -[case testNewTypeBadInitialization] -from typing import NewType - -a = NewType('b', int) -b = NewType('b', 3) -c = NewType(2, int) -foo = "d" -d = NewType(foo, int) -e = NewType(name='e', tp=int) -f = NewType('f', tp=int) -[out] -main:3: error: Argument 1 to NewType(...) does not match variable name -main:4: error: Argument 2 to NewType(...) must be a valid type -main:5: error: Argument 1 to NewType(...) must be a string literal -main:7: error: Argument 1 to NewType(...) must be a string literal -main:8: error: Argument 1 to NewType(...) must be a positional string literal -main:8: error: Argument 2 to NewType(...) must be a positional type -main:9: error: Argument 2 to NewType(...) must be a positional type - -[case testNewTypeWithCasts] -from typing import NewType, cast -UserId = NewType('UserId', int) -foo = UserId(3) -foo = cast(UserId, 3) -foo = cast(UserId, "foo") -foo = cast(UserId, UserId(4)) -[out] - [case testNewTypeWithCompositeTypes] from typing import NewType, Tuple, List TwoTuple = NewType('TwoTuple', Tuple[int, str]) @@ -94,11 +68,6 @@ reveal_type(baz) # E: Revealed type is 'builtins.list[__main__.UserId*]' [builtins fixtures/tuple.py] [out] -[case testNewTypeWithUnions] -from typing import NewType, Union -Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) -[out] - [case testNewTypeWithGenerics] from typing import TypeVar, Generic, NewType, Any @@ -126,21 +95,43 @@ reveal_type(Derived1(Base('a')).getter()) # E: Revealed type is 'builtins.str*' reveal_type(Derived3(Base('a')).getter()) # E: Revealed type is 'Any' [out] -[case testNewTypeInMultipleFiles] -import a -import b -list1 = [a.UserId(1), a.UserId(2)] -list1.append(b.UserId(3)) # E: Argument 1 to "append" of "list" has incompatible type "b.UserId"; expected "a.UserId" +[case testNewTypeWithNamedTuple] +from collections import namedtuple +from typing import NewType, NamedTuple -[file a.py] -from typing import NewType -UserId = NewType('UserId', int) +Vector1 = namedtuple('Vector1', ['x', 'y', 'z']) +Point1 = NewType('Point1', Vector1) +p1 = Point1(Vector1(1, 2, 3)) +reveal_type(p1.x) # E: Revealed type is 'Any' +reveal_type(p1.y) # E: Revealed type is 'Any' +reveal_type(p1.z) # E: Revealed type is 'Any' -[file b.py] -from typing import NewType -UserId = NewType('UserId', int) +Vector2 = NamedTuple('Vector2', [('x', int), ('y', int), ('z', int)]) +Point2 = NewType('Point2', Vector2) +p2 = Point2(Vector2(1, 2, 3)) +reveal_type(p2.x) # E: Revealed type is 'builtins.int' +reveal_type(p2.y) # E: Revealed type is 'builtins.int' +reveal_type(p2.z) # E: Revealed type is 'builtins.int' -[builtins fixtures/list.py] +class Vector3: + def __init__(self, x: int, y: int, z: int) -> None: + self.x = x + self.y = y + self.z = z +Point3 = NewType('Point3', Vector3) +p3 = Point3(Vector3(1, 3, 3)) +reveal_type(p3.x) # E: Revealed type is 'builtins.int' +reveal_type(p3.y) # E: Revealed type is 'builtins.int' +reveal_type(p3.z) # E: Revealed type is 'builtins.int' +[out] + +[case testNewTypeWithCasts] +from typing import NewType, cast +UserId = NewType('UserId', int) +foo = UserId(3) +foo = cast(UserId, 3) +foo = cast(UserId, "foo") +foo = cast(UserId, UserId(4)) [out] [case testNewTypeWithTypeAliases] @@ -153,42 +144,10 @@ def func1(x: Foo) -> Bar: def func2(x: int) -> Bar: return Bar(x) - [out] -[case testNewTypeWithTypeType] -from typing import NewType, Type -Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got Type[builtins.int]) -a = Foo(type(3)) -[builtins fixtures/args.py] -[out] -[case testNewTypeWithTypeVars] -from typing import NewType, TypeVar, List -T = TypeVar('T') -A = NewType('A', T) -B = NewType('B', List[T]) -[builtins fixtures/list.py] -[out] -main:3: error: Invalid type "__main__.T" -main:3: error: Argument 2 to NewType(...) must be subclassable (got T?) -main:4: error: Invalid type "__main__.T" - -[case testNewTypeRedefiningVariables] -from typing import NewType - -a = 3 -a = NewType('a', int) - -b = NewType('b', int) -b = NewType('b', float) - -c = NewType('c', str) # type: str -[out] -main:4: error: Cannot redefine 'a' as a NewType -main:7: error: Invalid assignment target -main:7: error: Cannot redefine 'b' as a NewType -main:9: error: Cannot declare the type of a NewType declaration +-- Make sure NewType works as expected in a variety of different scopes/across files [case testNewTypeInLocalScope] from typing import NewType @@ -215,53 +174,24 @@ c = MyClass.C(3.5) main: note: In function "func": main:9: error: Argument 1 to "A" has incompatible type "int"; expected "str" -[case testNewTypeTestSubclassingFails] -from typing import NewType -class A: pass -B = NewType('B', A) -class C(B): pass # E: Cannot subclass NewType -[out] +[case testNewTypeInMultipleFiles] +import a +import b +list1 = [a.UserId(1), a.UserId(2)] +list1.append(b.UserId(3)) # E: Argument 1 to "append" of "list" has incompatible type "b.UserId"; expected "a.UserId" -[case testNewTypeOfANewTypeFails] +[file a.py] from typing import NewType -A = NewType('A', int) -B = NewType('B', A) # E: Argument 2 to NewType(...) cannot be another NewType -C = A -D = C -E = NewType('E', D) # E: Argument 2 to NewType(...) cannot be another NewType -[out] - -[case testNewTypeWithNamedTuple] -from collections import namedtuple -from typing import NewType, NamedTuple - -Vector1 = namedtuple('Vector1', ['x', 'y', 'z']) -Point1 = NewType('Point1', Vector1) -p1 = Point1(Vector1(1, 2, 3)) -reveal_type(p1.x) # E: Revealed type is 'Any' -reveal_type(p1.y) # E: Revealed type is 'Any' -reveal_type(p1.z) # E: Revealed type is 'Any' +UserId = NewType('UserId', int) -Vector2 = NamedTuple('Vector2', [('x', int), ('y', int), ('z', int)]) -Point2 = NewType('Point2', Vector2) -p2 = Point2(Vector2(1, 2, 3)) -reveal_type(p2.x) # E: Revealed type is 'builtins.int' -reveal_type(p2.y) # E: Revealed type is 'builtins.int' -reveal_type(p2.z) # E: Revealed type is 'builtins.int' +[file b.py] +from typing import NewType +UserId = NewType('UserId', int) -class Vector3: - def __init__(self, x: int, y: int, z: int) -> None: - self.x = x - self.y = y - self.z = z -Point3 = NewType('Point3', Vector3) -p3 = Point3(Vector3(1, 3, 3)) -reveal_type(p3.x) # E: Revealed type is 'builtins.int' -reveal_type(p3.y) # E: Revealed type is 'builtins.int' -reveal_type(p3.z) # E: Revealed type is 'builtins.int' +[builtins fixtures/list.py] [out] -[case testIncrementalWithNewType] +[case testNewTypeWithIncremental] import m [file m.py] @@ -297,3 +227,80 @@ reveal_type(num) main:1: note: In module imported here: tmp/m.py:13: error: Revealed type is 'm.UserId' tmp/m.py:14: error: Revealed type is 'builtins.int' + + +-- Check misuses of NewType fail + +[case testNewTypeBadInitializationFails] +from typing import NewType + +a = NewType('b', int) +b = NewType('b', 3) +c = NewType(2, int) +foo = "d" +d = NewType(foo, int) +e = NewType(name='e', tp=int) +f = NewType('f', tp=int) +[out] +main:3: error: Argument 1 to NewType(...) does not match variable name +main:4: error: Argument 2 to NewType(...) must be a valid type +main:5: error: Argument 1 to NewType(...) must be a string literal +main:7: error: Argument 1 to NewType(...) must be a string literal +main:8: error: Argument 1 to NewType(...) must be a positional string literal +main:8: error: Argument 2 to NewType(...) must be a positional type +main:9: error: Argument 2 to NewType(...) must be a positional type + +[case testNewTypeWithUnionsFails] +from typing import NewType, Union +Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) +[out] + +[case testNewTypeWithTypeTypeFails] +from typing import NewType, Type +Foo = NewType('Foo', Type[int]) # E: Argument 2 to NewType(...) must be subclassable (got Type[builtins.int]) +a = Foo(type(3)) +[builtins fixtures/args.py] +[out] + +[case testNewTypeWithTypeVarsFails] +from typing import NewType, TypeVar, List +T = TypeVar('T') +A = NewType('A', T) +B = NewType('B', List[T]) +[builtins fixtures/list.py] +[out] +main:3: error: Invalid type "__main__.T" +main:3: error: Argument 2 to NewType(...) must be subclassable (got T?) +main:4: error: Invalid type "__main__.T" + +[case testNewTypeWithNewTypeFails] +from typing import NewType +A = NewType('A', int) +B = NewType('B', A) # E: Argument 2 to NewType(...) cannot be another NewType +C = A +D = C +E = NewType('E', D) # E: Argument 2 to NewType(...) cannot be another NewType +[out] + +[case testNewTypeRedefiningVariablesFails] +from typing import NewType + +a = 3 +a = NewType('a', int) + +b = NewType('b', int) +b = NewType('b', float) + +c = NewType('c', str) # type: str +[out] +main:4: error: Cannot redefine 'a' as a NewType +main:7: error: Invalid assignment target +main:7: error: Cannot redefine 'b' as a NewType +main:9: error: Cannot declare the type of a NewType declaration + +[case testNewTypeTestSubclassingFails] +from typing import NewType +class A: pass +B = NewType('B', A) +class C(B): pass # E: Cannot subclass NewType +[out] From d76b8daf414cee3a0dd02471564cb09e9a1c6314 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 09:09:42 -0700 Subject: [PATCH 17/26] Add test for NewType and Any --- test-data/unit/check-newtype.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 94de48f3455b..9cf01076c10f 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -250,6 +250,11 @@ main:8: error: Argument 1 to NewType(...) must be a positional string literal main:8: error: Argument 2 to NewType(...) must be a positional type main:9: error: Argument 2 to NewType(...) must be a positional type +[case testNewTypeWithAnyFails] +from typing import NewType, Any +A = NewType('A', Any) # E: Argument 2 to NewType(...) must be subclassable (got Any) +[out] + [case testNewTypeWithUnionsFails] from typing import NewType, Union Foo = NewType('Foo', Union[int, float]) # E: Argument 2 to NewType(...) must be subclassable (got Union[builtins.int, builtins.float]) From c81477db23223138f49b7aa7b20e829bcf5fe9ff Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 09:18:30 -0700 Subject: [PATCH 18/26] Rename variables and combine checks in semanal --- mypy/semanal.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 66649b5a10f4..22ec9a747ae3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1307,33 +1307,28 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: self.fail("Cannot redefine '%s' as a NewType" % name, s) return - underlying_type = self.check_newtype_args(call, name, s) - if underlying_type is None: + old_type = self.check_newtype_args(call, name, s) + if old_type is None: return - # Check if class is subtypeable - if isinstance(underlying_type, TupleType): - base_type = underlying_type.fallback - elif isinstance(underlying_type, Instance): - base_type = underlying_type + # Create the corresponding class def if it's subtypeable... + if isinstance(old_type, TupleType): + newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type.fallback) + newtype_class_info.tuple_type = old_type + elif isinstance(old_type, Instance): + newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type) else: message = "Argument 2 to NewType(...) must be subclassable (got {})" - self.fail(message.format(underlying_type), s) + self.fail(message.format(old_type), s) return - # Create the corresponding class def... - newtype_class_info = self.build_newtype_typeinfo(name, underlying_type, base_type) - # ...and add it to the symbol table. node = self.lookup(name, s) node.kind = GDEF # TODO: locally defined newtype call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) node.node = newtype_class_info - def build_newtype_typeinfo(self, - name: str, - underlying_type: Type, - base_type: Instance) -> TypeInfo: + def build_newtype_typeinfo(self,name: str, old_type: Type, base_type: Instance) -> TypeInfo: class_def = ClassDef(name, Block([])) class_def.fullname = self.qualified_name(name) @@ -1342,17 +1337,15 @@ def build_newtype_typeinfo(self, info.mro = [info] + base_type.type.mro info.bases = [base_type] info.is_newtype = True - if isinstance(underlying_type, TupleType): - info.tuple_type = underlying_type # Add __init__ method args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), - self.make_argument('item', underlying_type)] + self.make_argument('item', old_type)] signature = CallableType( - arg_types = [cast(Type, None), underlying_type], + arg_types = [cast(Type, None), old_type], arg_kinds = [arg.kind for arg in args], arg_names = ['self', 'item'], - ret_type = underlying_type, + ret_type = old_type, fallback = self.named_type('__builtins__.function'), name = name) init_func = FuncDef('__init__', args, Block([]), typ=signature) From 0995a30f839e79fe375b338bd9d9712720bad8da Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 09:45:50 -0700 Subject: [PATCH 19/26] Refactor newtype code in semenal --- mypy/semanal.py | 112 ++++++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 22ec9a747ae3..df6fc799f99e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1293,25 +1293,21 @@ def store_declared_types(self, lvalue: Node, typ: Type) -> None: def process_newtype_declaration(self, s: AssignmentStmt) -> None: """Check if s declares a NewType; if yes, store it in symbol table.""" + # Extract and check all information from newtype declaration call = self.get_newtype_declaration(s) - if not call: + if call is None: return call.analyzed = NewTypeExpr(None).set_line(call.line) - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a NewType declaration", s) - else: - self.fail("Cannot redefine '%s' as a NewType" % name, s) + name = self.get_newtype_name(s) + if name is None: return old_type = self.check_newtype_args(call, name, s) if old_type is None: return - # Create the corresponding class def if it's subtypeable... + # Create the corresponding class definition if the aliased type is subtypeable if isinstance(old_type, TupleType): newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type.fallback) newtype_class_info.tuple_type = old_type @@ -1322,39 +1318,41 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: self.fail(message.format(old_type), s) return - # ...and add it to the symbol table. + # If so, add it to the symbol table. node = self.lookup(name, s) - node.kind = GDEF # TODO: locally defined newtype - call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) + # TODO: why does NewType work in local scopes despite always being of kind GDEF? + node.kind = GDEF node.node = newtype_class_info + call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) - def build_newtype_typeinfo(self,name: str, old_type: Type, base_type: Instance) -> TypeInfo: - class_def = ClassDef(name, Block([])) - class_def.fullname = self.qualified_name(name) - - symbols = SymbolTable() - info = TypeInfo(symbols, class_def) - info.mro = [info] + base_type.type.mro - info.bases = [base_type] - info.is_newtype = True - - # Add __init__ method - args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), - self.make_argument('item', old_type)] - signature = CallableType( - arg_types = [cast(Type, None), old_type], - arg_kinds = [arg.kind for arg in args], - arg_names = ['self', 'item'], - ret_type = old_type, - fallback = self.named_type('__builtins__.function'), - name = name) - init_func = FuncDef('__init__', args, Block([]), typ=signature) - init_func.info = info - symbols['__init__'] = SymbolTableNode(MDEF, init_func) + def get_newtype_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: + """Returns the Newtype() call statement if `s` is a newtype declaration + or None otherwise.""" + # TODO: determine if this and get_typevar_declaration should be refactored + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return None + if not isinstance(s.rvalue, CallExpr): + return None + call = s.rvalue + if not isinstance(call.callee, RefExpr): + return None + callee = call.callee + if callee.fullname != 'typing.NewType': + return None + return call - return info + def get_newtype_name(self, s: AssignmentStmt) -> Optional[str]: + lvalue = cast(NameExpr, s.lvalues[0]) + name = lvalue.name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a NewType declaration", s) + else: + self.fail("Cannot redefine '%s' as a NewType" % name, s) + return None + return name - def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Type: + def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Optional[Type]: has_failed = False args = call.args if len(args) != 2: @@ -1387,21 +1385,31 @@ def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Typ return None if has_failed else value - def get_newtype_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the Newtype() call statement if `s` is a newtype declaration - or None otherwise.""" - # TODO: determine if this and get_typevar_declaration should be refactored - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.NewType': - return None - return call + def build_newtype_typeinfo(self, name: str, old_type: Type, base_type: Instance) -> TypeInfo: + class_def = ClassDef(name, Block([])) + class_def.fullname = self.qualified_name(name) + + symbols = SymbolTable() + info = TypeInfo(symbols, class_def) + info.mro = [info] + base_type.type.mro + info.bases = [base_type] + info.is_newtype = True + + # Add __init__ method + args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), + self.make_argument('item', old_type)] + signature = CallableType( + arg_types = [cast(Type, None), old_type], + arg_kinds = [arg.kind for arg in args], + arg_names = ['self', 'item'], + ret_type = old_type, + fallback = self.named_type('__builtins__.function'), + name = name) + init_func = FuncDef('__init__', args, Block([]), typ=signature) + init_func.info = info + symbols['__init__'] = SymbolTableNode(MDEF, init_func) + + return info def process_typevar_declaration(self, s: AssignmentStmt) -> None: """Check if s declares a TypeVar; it yes, store it in symbol table.""" From c37520b6f27d1eab9ed93ca8bc4b2208d9de6323 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:10:47 -0700 Subject: [PATCH 20/26] Clarify info in documentation --- docs/source/kinds_of_types.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index 99efcf9a8af0..ba36541f37f7 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -457,7 +457,12 @@ definition: ... However, at runtime, ``NewType('Derived', Base)`` will return a dummy function that -simply returns its argument. +simply returns its argument: + +.. code-block:: python + + def Derived(_x: Base) -> Base: + return _x Mypy will require explicit casts from ``int`` where ``UserId`` is expected, while implicitly casting from ``UserId`` where ``int`` is expected. Examples: @@ -478,10 +483,14 @@ implicitly casting from ``UserId`` where ``int`` is expected. Examples: num = UserId(5) + 1 # type: int -``NewType`` accepts only one argument that must be a properly subclassable class, i.e., -not a type construct like ``Union``, etc. The function returned by ``NewType`` accepts -only one argument; this is equivalent to supporting only one constructor accepting an -instance of the base class (see above). Example: +``NewType`` accepts exactly two arguments. The first argument must be a string containing +the name of the new type and must equal the name of the variable to which the new type is +assigned. The second argument must be a properly subclassable class, i.e., +not a type construct like ``Union``, etc. + +The function returned by ``NewType`` accepts only one argument; this is equivalent to +supporting only one constructor accepting an instance of the base class (see above). +Example: .. code-block:: python From 0eb700aa68cd432f60fed35abcec601bdc4f97a3 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:11:14 -0700 Subject: [PATCH 21/26] Remove unnecessary extra code --- mypy/fixup.py | 5 +---- mypy/nodes.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 80fdb71568ab..929da2426508 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -4,7 +4,7 @@ from mypy.nodes import (MypyFile, SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, FuncDef, OverloadedFuncDef, Decorator, Var, - TypeVarExpr, NewTypeExpr, ClassDef, + TypeVarExpr, ClassDef, LDEF, MDEF, GDEF, MODULE_REF) from mypy.types import (CallableType, EllipsisType, Instance, Overloaded, TupleType, TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor, @@ -123,9 +123,6 @@ def visit_type_var_expr(self, tv: TypeVarExpr) -> None: value.accept(self.type_fixer) tv.upper_bound.accept(self.type_fixer) - def visit_newtype_expr(self, nt: NewTypeExpr) -> None: - raise NotImplemented() - def visit_var(self, v: Var) -> None: if self.current_info is not None: v.info = self.current_info diff --git a/mypy/nodes.py b/mypy/nodes.py index 769e42e88534..86942b03b28e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1989,7 +1989,7 @@ def serialize(self) -> Union[str, JsonDict]: '_promote': None if self._promote is None else self._promote.serialize(), 'tuple_type': None if self.tuple_type is None else self.tuple_type.serialize(), 'is_named_tuple': self.is_named_tuple, - 'is_newtype': self.is_newtype + 'is_newtype': self.is_newtype, } return data From 90609af36ad0db0ee6b94deec947d19bc366f1f2 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:11:24 -0700 Subject: [PATCH 22/26] Refactor semanal code --- mypy/semanal.py | 86 ++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index df6fc799f99e..56b982242d5f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -71,7 +71,7 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, TypeType, Void, + FunctionLike, UnboundType, TypeList, ErrorType, TypeVarDef, Void, replace_leading_arg_type, TupleType, UnionType, StarType, EllipsisType ) from mypy.nodes import function_type, implicit_module_attrs @@ -1294,16 +1294,11 @@ def store_declared_types(self, lvalue: Node, typ: Type) -> None: def process_newtype_declaration(self, s: AssignmentStmt) -> None: """Check if s declares a NewType; if yes, store it in symbol table.""" # Extract and check all information from newtype declaration - call = self.get_newtype_declaration(s) - if call is None: + name, call = self.analyze_newtype_declaration(s) + if name is None or call is None: return - call.analyzed = NewTypeExpr(None).set_line(call.line) - name = self.get_newtype_name(s) - if name is None: - return - - old_type = self.check_newtype_args(call, name, s) + old_type = self.check_newtype_args(name, call, s) if old_type is None: return @@ -1320,70 +1315,67 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None: # If so, add it to the symbol table. node = self.lookup(name, s) + if node is None: + self.fail("Could not find {} in current namespace".format(name), s) + return # TODO: why does NewType work in local scopes despite always being of kind GDEF? node.kind = GDEF node.node = newtype_class_info call.analyzed = NewTypeExpr(newtype_class_info).set_line(call.line) - def get_newtype_declaration(self, s: AssignmentStmt) -> Optional[CallExpr]: - """Returns the Newtype() call statement if `s` is a newtype declaration - or None otherwise.""" - # TODO: determine if this and get_typevar_declaration should be refactored - if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): - return None - if not isinstance(s.rvalue, CallExpr): - return None - call = s.rvalue - if not isinstance(call.callee, RefExpr): - return None - callee = call.callee - if callee.fullname != 'typing.NewType': - return None - return call + def analyze_newtype_declaration(self, + s: AssignmentStmt) -> Tuple[Optional[str], Optional[CallExpr]]: + """Return the NewType call expression if `s` is a newtype declaration or None otherwise.""" + name, call = None, None + if (len(s.lvalues) == 1 + and isinstance(s.lvalues[0], NameExpr) + and isinstance(s.rvalue, CallExpr) + and isinstance(s.rvalue.callee, RefExpr) + and s.rvalue.callee.fullname == 'typing.NewType'): + lvalue = s.lvalues[0] + name = s.lvalues[0].name + if not lvalue.is_def: + if s.type: + self.fail("Cannot declare the type of a NewType declaration", s) + else: + self.fail("Cannot redefine '%s' as a NewType" % name, s) - def get_newtype_name(self, s: AssignmentStmt) -> Optional[str]: - lvalue = cast(NameExpr, s.lvalues[0]) - name = lvalue.name - if not lvalue.is_def: - if s.type: - self.fail("Cannot declare the type of a NewType declaration", s) - else: - self.fail("Cannot redefine '%s' as a NewType" % name, s) - return None - return name + # This dummy NewTypeExpr marks the call as sufficiently analyzed; it will be + # overwritten later with a fully complete NewTypeExpr if there are no other + # errors with the NewType() call. + call = s.rvalue + call.analyzed = NewTypeExpr(None).set_line(call.line) - def check_newtype_args(self, call: CallExpr, name: str, context: Context) -> Optional[Type]: + return name, call + + def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Optional[Type]: has_failed = False - args = call.args - if len(args) != 2: - self.fail("NewType(...) expects exactly two arguments", context) + args, arg_kinds = call.args, call.arg_kinds + if len(args) != 2 or arg_kinds[0] != ARG_POS or arg_kinds[1] != ARG_POS: + self.fail("NewType(...) expects exactly two positional arguments", context) return None # Check first argument if not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)): self.fail("Argument 1 to NewType(...) must be a string literal", context) has_failed = True - elif call.arg_kinds[0] != ARG_POS: - self.fail("Argument 1 to NewType(...) must be a positional string literal", context) - has_failed = True elif cast(StrExpr, call.args[0]).value != name: self.fail("Argument 1 to NewType(...) does not match variable name", context) has_failed = True # Check second argument try: - value = self.anal_type(expr_to_unanalyzed_type(call.args[1])) + unanalyzed_type = expr_to_unanalyzed_type(call.args[1]) except TypeTranslationError: self.fail("Argument 2 to NewType(...) must be a valid type", context) return None + old_type = self.anal_type(unanalyzed_type) - if call.arg_kinds[1] != ARG_POS: - self.fail("Argument 2 to NewType(...) must be a positional type", context) - has_failed = True - elif isinstance(value, Instance) and value.type.is_newtype: + if isinstance(old_type, Instance) and old_type.type.is_newtype: self.fail("Argument 2 to NewType(...) cannot be another NewType", context) + has_failed = True - return None if has_failed else value + return None if has_failed else old_type def build_newtype_typeinfo(self, name: str, old_type: Type, base_type: Instance) -> TypeInfo: class_def = ClassDef(name, Block([])) From 578c8453db28f5cd1b057fe367f678404d1c8b01 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:13:33 -0700 Subject: [PATCH 23/26] Fix lint errors --- mypy/semanal.py | 14 +++++++------- test-data/unit/lib-stub/typing.py | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 56b982242d5f..a58a522e718f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1347,7 +1347,7 @@ def analyze_newtype_declaration(self, call.analyzed = NewTypeExpr(None).set_line(call.line) return name, call - + def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Optional[Type]: has_failed = False args, arg_kinds = call.args, call.arg_kinds @@ -1391,12 +1391,12 @@ def build_newtype_typeinfo(self, name: str, old_type: Type, base_type: Instance) args = [Argument(Var('cls'), NoneTyp(), None, ARG_POS), self.make_argument('item', old_type)] signature = CallableType( - arg_types = [cast(Type, None), old_type], - arg_kinds = [arg.kind for arg in args], - arg_names = ['self', 'item'], - ret_type = old_type, - fallback = self.named_type('__builtins__.function'), - name = name) + arg_types=[cast(Type, None), old_type], + arg_kinds=[arg.kind for arg in args], + arg_names=['self', 'item'], + ret_type=old_type, + fallback=self.named_type('__builtins__.function'), + name=name) init_func = FuncDef('__init__', args, Block([]), typ=signature) init_func.info = info symbols['__init__'] = SymbolTableNode(MDEF, init_func) diff --git a/test-data/unit/lib-stub/typing.py b/test-data/unit/lib-stub/typing.py index 77fb3a751c5a..85875d89a432 100644 --- a/test-data/unit/lib-stub/typing.py +++ b/test-data/unit/lib-stub/typing.py @@ -78,4 +78,3 @@ def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: def new_type(x): return x return new_type - From b2732503b4b194b84f39659ac1c161b89f7e501e Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:15:49 -0700 Subject: [PATCH 24/26] Fix tests based on new error message --- test-data/unit/check-newtype.test | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 9cf01076c10f..d1934dc362a0 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -246,9 +246,8 @@ main:3: error: Argument 1 to NewType(...) does not match variable name main:4: error: Argument 2 to NewType(...) must be a valid type main:5: error: Argument 1 to NewType(...) must be a string literal main:7: error: Argument 1 to NewType(...) must be a string literal -main:8: error: Argument 1 to NewType(...) must be a positional string literal -main:8: error: Argument 2 to NewType(...) must be a positional type -main:9: error: Argument 2 to NewType(...) must be a positional type +main:8: error: NewType(...) expects exactly two positional arguments +main:9: error: NewType(...) expects exactly two positional arguments [case testNewTypeWithAnyFails] from typing import NewType, Any From c71aafaa485f6afe0bc9a69f92c0c0b825b75704 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 18:26:31 -0700 Subject: [PATCH 25/26] Update tests for newtype to respond to code review --- test-data/unit/check-newtype.test | 65 ++++++++++++++++++------------- test-data/unit/fixtures/list.py | 1 + test-data/unit/fixtures/tuple.py | 12 ------ 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index d1934dc362a0..59c36690777d 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -39,15 +39,19 @@ tcp_packet = TcpPacketId(127, 0) main:12: error: Too many arguments for "TcpPacketId" main:12: error: Argument 1 to "TcpPacketId" has incompatible type "int"; expected "PacketId" -[case testNewTypeWithCompositeTypes] -from typing import NewType, Tuple, List +[case testNewTypeWithTuples] +from typing import NewType, Tuple TwoTuple = NewType('TwoTuple', Tuple[int, str]) a = TwoTuple((3, "a")) b = TwoTuple(("a", 3)) # E: Argument 1 to "TwoTuple" has incompatible type "Tuple[str, int]"; expected "Tuple[int, str]" reveal_type(a[0]) # E: Revealed type is 'builtins.int' reveal_type(a[1]) # E: Revealed type is 'builtins.str' +[builtins fixtures/tuple.py] +[out] +[case testNewTypeWithLists] +from typing import NewType, List UserId = NewType('UserId', int) IdList = NewType('IdList', List[UserId]) @@ -65,7 +69,7 @@ reveal_type(foo) # E: Revealed type is '__main__.IdList' reveal_type(bar) # E: Revealed type is '__main__.IdList' reveal_type(baz) # E: Revealed type is 'builtins.list[__main__.UserId*]' -[builtins fixtures/tuple.py] +[builtins fixtures/list.py] [out] [case testNewTypeWithGenerics] @@ -99,30 +103,26 @@ reveal_type(Derived3(Base('a')).getter()) # E: Revealed type is 'Any' from collections import namedtuple from typing import NewType, NamedTuple -Vector1 = namedtuple('Vector1', ['x', 'y', 'z']) +Vector1 = namedtuple('Vector1', ['x', 'y']) Point1 = NewType('Point1', Vector1) -p1 = Point1(Vector1(1, 2, 3)) +p1 = Point1(Vector1(1, 2)) reveal_type(p1.x) # E: Revealed type is 'Any' reveal_type(p1.y) # E: Revealed type is 'Any' -reveal_type(p1.z) # E: Revealed type is 'Any' -Vector2 = NamedTuple('Vector2', [('x', int), ('y', int), ('z', int)]) +Vector2 = NamedTuple('Vector2', [('x', int), ('y', int)]) Point2 = NewType('Point2', Vector2) -p2 = Point2(Vector2(1, 2, 3)) +p2 = Point2(Vector2(1, 2)) reveal_type(p2.x) # E: Revealed type is 'builtins.int' reveal_type(p2.y) # E: Revealed type is 'builtins.int' -reveal_type(p2.z) # E: Revealed type is 'builtins.int' class Vector3: - def __init__(self, x: int, y: int, z: int) -> None: + def __init__(self, x: int, y: int) -> None: self.x = x self.y = y - self.z = z Point3 = NewType('Point3', Vector3) -p3 = Point3(Vector3(1, 3, 3)) +p3 = Point3(Vector3(1, 3)) reveal_type(p3.x) # E: Revealed type is 'builtins.int' reveal_type(p3.y) # E: Revealed type is 'builtins.int' -reveal_type(p3.z) # E: Revealed type is 'builtins.int' [out] [case testNewTypeWithCasts] @@ -138,12 +138,21 @@ foo = cast(UserId, UserId(4)) from typing import NewType Foo = int Bar = NewType('Bar', Foo) +Bar2 = Bar def func1(x: Foo) -> Bar: return Bar(x) def func2(x: int) -> Bar: return Bar(x) + +def func3(x: Bar2) -> Bar: + return x + +x = Bar(42) +y = Bar2(42) + +y = func3(x) [out] @@ -158,7 +167,7 @@ def func() -> None: A = NewType('A', str) B = NewType('B', str) - a = A(3) + a = A(3) # E: Argument 1 to "A" has incompatible type "int"; expected "str" a = A('xyz') b = B('xyz') @@ -172,7 +181,6 @@ b = A(3) c = MyClass.C(3.5) [out] main: note: In function "func": -main:9: error: Argument 1 to "A" has incompatible type "int"; expected "str" [case testNewTypeInMultipleFiles] import a @@ -234,20 +242,14 @@ tmp/m.py:14: error: Revealed type is 'builtins.int' [case testNewTypeBadInitializationFails] from typing import NewType -a = NewType('b', int) -b = NewType('b', 3) -c = NewType(2, int) +a = NewType('b', int) # E: Argument 1 to NewType(...) does not match variable name +b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type +c = NewType(2, int) # E: Argument 1 to NewType(...) must be a string literal foo = "d" -d = NewType(foo, int) -e = NewType(name='e', tp=int) -f = NewType('f', tp=int) +d = NewType(foo, int) # E: Argument 1 to NewType(...) must be a string literal +e = NewType(name='e', tp=int) # E: NewType(...) expects exactly two positional arguments +f = NewType('f', tp=int) # E: NewType(...) expects exactly two positional arguments [out] -main:3: error: Argument 1 to NewType(...) does not match variable name -main:4: error: Argument 2 to NewType(...) must be a valid type -main:5: error: Argument 1 to NewType(...) must be a string literal -main:7: error: Argument 1 to NewType(...) must be a string literal -main:8: error: NewType(...) expects exactly two positional arguments -main:9: error: NewType(...) expects exactly two positional arguments [case testNewTypeWithAnyFails] from typing import NewType, Any @@ -293,7 +295,7 @@ a = 3 a = NewType('a', int) b = NewType('b', int) -b = NewType('b', float) +b = NewType('b', float) # this line throws two errors c = NewType('c', str) # type: str [out] @@ -302,6 +304,13 @@ main:7: error: Invalid assignment target main:7: error: Cannot redefine 'b' as a NewType main:9: error: Cannot declare the type of a NewType declaration +[case testNewTypeAddingExplicitTypesFails] +from typing import NewType +UserId = NewType('UserId', int) + +a = 3 # type: UserId # E: Incompatible types in assignment (expression has type "int", variable has type "UserId") +[out] + [case testNewTypeTestSubclassingFails] from typing import NewType class A: pass diff --git a/test-data/unit/fixtures/list.py b/test-data/unit/fixtures/list.py index 55c8999cb46a..220ab529b818 100644 --- a/test-data/unit/fixtures/list.py +++ b/test-data/unit/fixtures/list.py @@ -16,6 +16,7 @@ def __init__(self) -> None: pass @overload def __init__(self, x: Iterable[T]) -> None: pass def __iter__(self) -> Iterator[T]: pass + def __add__(self, x: list[T]) -> list[T]: pass def __mul__(self, x: int) -> list[T]: pass def __getitem__(self, x: int) -> T: pass def append(self, x: T) -> None: pass diff --git a/test-data/unit/fixtures/tuple.py b/test-data/unit/fixtures/tuple.py index 2d0cab5b6f02..fa3636764fb9 100644 --- a/test-data/unit/fixtures/tuple.py +++ b/test-data/unit/fixtures/tuple.py @@ -24,15 +24,3 @@ class str: pass # For convenience def sum(iterable: Iterable[T], start: T = None) -> T: pass True = bool() - -class list(Iterable[T], Generic[T]): - @overload - def __init__(self) -> None: pass - @overload - def __init__(self, x: Iterable[T]) -> None: pass - def __iter__(self) -> Iterator[T]: pass - def __mul__(self, x: int) -> list[T]: pass - def __add__(self, x: list[T]) -> list[T]: pass - def __getitem__(self, x: int) -> T: pass - def append(self, x: T) -> None: pass - def extend(self, x: Iterable[T]) -> None: pass From 55617b9f9908023f4cd544cfa1f9bf056d63ffcd Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 27 Jul 2016 20:10:28 -0700 Subject: [PATCH 26/26] Respond to 2nd wave of code review --- docs/source/kinds_of_types.rst | 8 ++++---- test-data/unit/fixtures/tuple.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index ba36541f37f7..16c17e1d888d 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -461,7 +461,7 @@ simply returns its argument: .. code-block:: python - def Derived(_x: Base) -> Base: + def Derived(_x): return _x Mypy will require explicit casts from ``int`` where ``UserId`` is expected, while @@ -483,9 +483,9 @@ implicitly casting from ``UserId`` where ``int`` is expected. Examples: num = UserId(5) + 1 # type: int -``NewType`` accepts exactly two arguments. The first argument must be a string containing -the name of the new type and must equal the name of the variable to which the new type is -assigned. The second argument must be a properly subclassable class, i.e., +``NewType`` accepts exactly two arguments. The first argument must be a string literal +containing the name of the new type and must equal the name of the variable to which the new +type is assigned. The second argument must be a properly subclassable class, i.e., not a type construct like ``Union``, etc. The function returned by ``NewType`` accepts only one argument; this is equivalent to diff --git a/test-data/unit/fixtures/tuple.py b/test-data/unit/fixtures/tuple.py index fa3636764fb9..76c109127364 100644 --- a/test-data/unit/fixtures/tuple.py +++ b/test-data/unit/fixtures/tuple.py @@ -1,6 +1,6 @@ # Builtins stub used in tuple-related test cases. -from typing import Iterable, Iterator, TypeVar, Generic, Sequence, overload +from typing import Iterable, TypeVar, Generic, Sequence Tco = TypeVar('Tco', covariant=True)