From f51cad039c8e4d59616925baa390c1bf798854ff Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Feb 2018 12:47:51 +0000 Subject: [PATCH 01/23] Merge unanalyzed_type attribute of FuncDefs Also process overloaded and non-overloaded functions more consistently in AST merge. --- mypy/server/astmerge.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 3d13edac14bc..ce4e94235de5 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -50,7 +50,7 @@ from mypy.nodes import ( Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo, FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr, - OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, MDEF + OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, MDEF ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -156,15 +156,11 @@ def visit_block(self, node: Block) -> None: def visit_func_def(self, node: FuncDef) -> None: node = self.fixup(node) - if node.type: - self.fixup_type(node.type) - if node.info: - node.info = self.fixup(node.info) + self.process_base_func(node) super().visit_func_def(node) def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None: - if node.info: - node.info = self.fixup(node.info) + self.process_base_func(node) super().visit_overloaded_func_def(node) def visit_class_def(self, node: ClassDef) -> None: @@ -176,6 +172,15 @@ def visit_class_def(self, node: ClassDef) -> None: self.process_type_info(node.info) super().visit_class_def(node) + def process_base_func(self, node: FuncBase) -> None: + if node.type: + self.fixup_type(node.type) + if node.info: + node.info = self.fixup(node.info) + if node.unanalyzed_type: + # Unanalyzed types can have AST node references + self.fixup_type(node.unanalyzed_type) + def process_type_var_def(self, tv: TypeVarDef) -> None: for value in tv.values: self.fixup_type(value) From 1da9ba393c0cc4d5aa0d062d20670a958f98ad4e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Feb 2018 14:37:19 +0000 Subject: [PATCH 02/23] Fix merge of Var info attributes inside TypeInfos --- mypy/server/astmerge.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index ce4e94235de5..9c649f21c7c1 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -377,14 +377,18 @@ def replace_nodes_in_symbol_table(symbols: SymbolTable, new.__dict__ = node.node.__dict__ node.node = new # TODO: Other node types - if isinstance(node.node, Var) and node.node.type: - node.node.type.accept(TypeReplaceVisitor(replacements)) + if isinstance(node.node, Var): + if node.node.type: + node.node.type.accept(TypeReplaceVisitor(replacements)) node.node.info = cast(TypeInfo, replacements.get(node.node.info, node.node.info)) else: # TODO: Other node types - if isinstance(node.node, Var) and node.node.type: - node.node.type.accept(TypeReplaceVisitor(replacements)) + if isinstance(node.node, Var): + if node.node.type: + node.node.type.accept(TypeReplaceVisitor(replacements)) + node.node.info = cast(TypeInfo, replacements.get(node.node.info, + node.node.info)) override = node.type_override if override: override.accept(TypeReplaceVisitor(replacements)) From 823445216fe93554cf5d13600a4b4e53343fa401 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 09:34:33 +0000 Subject: [PATCH 03/23] Fix TypeList types in AST merge They can appear in UnboundType instances. --- mypy/server/astmerge.py | 6 +++++- mypy/types.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 9c649f21c7c1..59fa4dbbd846 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -56,7 +56,7 @@ from mypy.types import ( Type, TypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, - Overloaded, TypeVarDef + Overloaded, TypeVarDef, TypeList ) from mypy.util import get_prefix @@ -350,6 +350,10 @@ def visit_unbound_type(self, typ: UnboundType) -> None: for arg in typ.args: arg.accept(self) + def visit_type_list(self, typ: TypeList) -> None: + for item in typ.items: + item.accept(self) + def visit_uninhabited_type(self, typ: UninhabitedType) -> None: pass diff --git a/mypy/types.py b/mypy/types.py index ca0f108d3ecc..226d7e998209 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -242,9 +242,10 @@ def serialize(self) -> JsonDict: class TypeList(Type): """Information about argument types and names [...]. - This is only used for the arguments of a Callable type, i.e. for + This is used for the arguments of a Callable type, i.e. for [arg, ...] in Callable[[arg, ...], ret]. This is not a real type - but a syntactic AST construct. + but a syntactic AST construct. UnboundTypes can also have TypeList + types before they are processed into Callable type.s """ items = None # type: List[Type] @@ -254,7 +255,6 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: self.items = items def accept(self, visitor: 'TypeVisitor[T]') -> T: - assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_type_list(self) def serialize(self) -> JsonDict: From 4d51e0cbd30c7341052f39d61177d3fa90e5ee58 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 09:40:01 +0000 Subject: [PATCH 04/23] Fix AST merge of TypeAliasExpr nodes Not sure yet if this actually fixes any user-visible issues. --- mypy/server/astmerge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 59fa4dbbd846..41f87a59cf68 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -50,7 +50,7 @@ from mypy.nodes import ( Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo, FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr, - OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, MDEF + OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, MDEF ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -238,6 +238,11 @@ def visit_enum_call_expr(self, node: EnumCallExpr) -> None: self.process_type_info(node.info) super().visit_enum_call_expr(node) + def visit_type_alias_expr(self, node: TypeAliasExpr) -> None: + self.fixup_type(node.type) + self.fixup_type(node.fallback) + super().visit_type_alias_expr(node) + # Others def visit_var(self, node: Var) -> None: From 0f95792a2662cd5c6daea4ba301eb97153607387 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 09:40:51 +0000 Subject: [PATCH 05/23] Fix AST merge of NewType nodes There were two issues: * The `info` attribute of the corresponding `ClassDef` wasn't updated. * `FuncDef` objects within the related `TypeInfo` weren't processed, since they are only accessible through a symbol table. Functions in normal classes are accessible by traversing the AST, in contrast. --- mypy/server/astmerge.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 41f87a59cf68..ca7959865e1a 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -220,10 +220,20 @@ def visit_newtype_expr(self, node: NewTypeExpr) -> None: if node.info: node.info = self.fixup(node.info) self.process_type_info(node.info) + self.process_synthetic_type_info(node.info) if node.old_type: self.fixup_type(node.old_type) super().visit_newtype_expr(node) + def process_synthetic_type_info(self, info: TypeInfo) -> None: + # Synthetic types (types not created using a class statement) don't + # have bodies in the AST so we need to iterate over their symbol + # tables separately, unlike normal classes. + info.defn.info = info + for name, node in info.names.items(): + if node.node: + node.node.accept(self) + def visit_lambda_expr(self, node: LambdaExpr) -> None: if node.info: node.info = self.fixup(node.info) From 49ea762c46cfc69521e81b7f68720a2c1453ee1f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 09:43:37 +0000 Subject: [PATCH 06/23] Fix leak of old MypyFile objects in AST merge This could at least leak memory. Not sure if this could cause invalid type checking results. --- mypy/server/update.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/server/update.py b/mypy/server/update.py index d8408c6c09a9..efc1224ae833 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -698,9 +698,11 @@ def replace_modules_with_new_variants( for id in new_modules: new_module = new_modules[id] if id in old_modules and new_module is not None: - merge_asts(old_modules[id], old_modules[id].names, + preserved_module = old_modules[id] + merge_asts(preserved_module, old_modules[id].names, new_module, new_module.names) - manager.modules[id] = old_modules[id] + manager.modules[id] = preserved_module + graph[id].tree = preserved_module def propagate_changes_using_dependencies( From c71dff952f0668bad09a310041b96e64451995c8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 12:21:26 +0000 Subject: [PATCH 07/23] Fix merge of TypeVarExpr --- mypy/server/astmerge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index ca7959865e1a..07529e572985 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -50,7 +50,8 @@ from mypy.nodes import ( Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo, FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr, - OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, MDEF + OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, CallExpr, + MDEF ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -216,6 +217,10 @@ def visit_super_expr(self, node: SuperExpr) -> None: if node.info is not None: node.info = self.fixup(node.info) + def visit_call_expr(self, node: CallExpr) -> None: + super().visit_call_expr(node) + node.analyzed = self.fixup(node.analyzed) + def visit_newtype_expr(self, node: NewTypeExpr) -> None: if node.info: node.info = self.fixup(node.info) From a170a98db29294961f2c6fbdf113ee048d199aa5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 13:02:10 +0000 Subject: [PATCH 08/23] Fix AST merge of newtypes --- mypy/server/astmerge.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 07529e572985..87cf2cd1ced9 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -224,21 +224,11 @@ def visit_call_expr(self, node: CallExpr) -> None: def visit_newtype_expr(self, node: NewTypeExpr) -> None: if node.info: node.info = self.fixup(node.info) - self.process_type_info(node.info) self.process_synthetic_type_info(node.info) if node.old_type: self.fixup_type(node.old_type) super().visit_newtype_expr(node) - def process_synthetic_type_info(self, info: TypeInfo) -> None: - # Synthetic types (types not created using a class statement) don't - # have bodies in the AST so we need to iterate over their symbol - # tables separately, unlike normal classes. - info.defn.info = info - for name, node in info.names.items(): - if node.node: - node.node.accept(self) - def visit_lambda_expr(self, node: LambdaExpr) -> None: if node.info: node.info = self.fixup(node.info) @@ -250,7 +240,7 @@ def visit_typeddict_expr(self, node: TypedDictExpr) -> None: def visit_enum_call_expr(self, node: EnumCallExpr) -> None: node.info = self.fixup(node.info) - self.process_type_info(node.info) + self.process_synthetic_type_info(node.info) super().visit_enum_call_expr(node) def visit_type_alias_expr(self, node: TypeAliasExpr) -> None: @@ -296,6 +286,16 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: if info.tuple_type: self.fixup_type(info.tuple_type) + def process_synthetic_type_info(self, info: TypeInfo) -> None: + # Synthetic types (types not created using a class statement) don't + # have bodies in the AST so we need to iterate over their symbol + # tables separately, unlike normal classes. + self.process_type_info(info) + info.defn.info = info + for name, node in info.names.items(): + if node.node: + node.node.accept(self) + def replace_statements(self, nodes: List[Statement]) -> List[Statement]: result = [] for node in nodes: From ebf131a4a8b2e0b030ee1e8a295f848549a29583 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 13:04:03 +0000 Subject: [PATCH 09/23] Fix AST merge of NamedTupleExpr --- mypy/server/astmerge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 87cf2cd1ced9..64b11b889ba0 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -210,7 +210,7 @@ def visit_ref_expr(self, node: RefExpr) -> None: def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None: super().visit_namedtuple_expr(node) node.info = self.fixup(node.info) - self.process_type_info(node.info) + self.process_synthetic_type_info(node.info) def visit_super_expr(self, node: SuperExpr) -> None: super().visit_super_expr(node) From 3d6a9c6e5dfe5cf65d072fe61a93db93cbb4d8f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 14:10:18 +0000 Subject: [PATCH 10/23] Fix AST traverser special case (ClassDef.analyzed) --- mypy/traverser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/traverser.py b/mypy/traverser.py index e8eaf5d4f9e1..978c184e6bdb 100644 --- a/mypy/traverser.py +++ b/mypy/traverser.py @@ -59,6 +59,8 @@ def visit_class_def(self, o: ClassDef) -> None: for base in o.base_type_exprs: base.accept(self) o.defs.accept(self) + if o.analyzed: + o.analyzed.accept(self) def visit_decorator(self, o: Decorator) -> None: o.func.accept(self) From 0f4902faf3670d9e2e9b300d804f4f50769ec7ee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Feb 2018 14:13:10 +0000 Subject: [PATCH 11/23] Fixes to AST merge of named tuples and typed dicts --- mypy/server/astmerge.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 64b11b889ba0..63bf6e640561 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -168,9 +168,14 @@ def visit_class_def(self, node: ClassDef) -> None: # TODO additional things? node.defs.body = self.replace_statements(node.defs.body) node.info = self.fixup(node.info) + info = node.info for tv in node.type_vars: self.process_type_var_def(tv) - self.process_type_info(node.info) + if info: + if info.is_named_tuple: + self.process_synthetic_type_info(info) + else: + self.process_type_info(info) super().visit_class_def(node) def process_base_func(self, node: FuncBase) -> None: @@ -235,8 +240,9 @@ def visit_lambda_expr(self, node: LambdaExpr) -> None: super().visit_lambda_expr(node) def visit_typeddict_expr(self, node: TypedDictExpr) -> None: - node.info = self.fixup(node.info) super().visit_typeddict_expr(node) + node.info = self.fixup(node.info) + self.process_synthetic_type_info(node.info) def visit_enum_call_expr(self, node: EnumCallExpr) -> None: node.info = self.fixup(node.info) @@ -276,8 +282,10 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: # - declared_metaclass # - metaclass_type # - _promote - # - typeddict_type # - replaced + info.defn.info = self.fixup(info) + if info.typeddict_type: + self.fixup_type(info.typeddict_type) replace_nodes_in_symbol_table(info.names, self.replacements) for i, item in enumerate(info.mro): info.mro[i] = self.fixup(info.mro[i]) @@ -291,7 +299,6 @@ def process_synthetic_type_info(self, info: TypeInfo) -> None: # have bodies in the AST so we need to iterate over their symbol # tables separately, unlike normal classes. self.process_type_info(info) - info.defn.info = info for name, node in info.names.items(): if node.node: node.node.accept(self) From 6aa24611d8df6788d8cb7db7924daefed8bd2f2d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 10:44:07 +0000 Subject: [PATCH 12/23] Don't retain stale AST nodes in semantic analyzer --- mypy/semanal.py | 6 ++++++ mypy/semanal_pass3.py | 2 ++ mypy/server/objgraph.py | 2 -- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 20b87237c54b..4f80258e89e1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -306,6 +306,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.options del self.patches + del self.cur_mod_node + del self.globals def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None: """Refresh a stale target in fine-grained incremental mode.""" @@ -324,6 +326,7 @@ def refresh_top_level(self, file_node: MypyFile) -> None: self.recurse_into_functions = False for d in file_node.defs: self.accept(d) + del self.patches @contextmanager def file_context(self, file_node: MypyFile, fnam: str, options: Options, @@ -988,6 +991,9 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: defn.name, items, types, default_items) node.node = info defn.info.replaced = info + # Mark that the original TypeInfo should no longer be used. + defn.info._fullname += '' + print('orig:', id(defn.info), 'replaced:', id(info)) defn.info = info defn.analyzed = NamedTupleExpr(info) defn.analyzed.line = defn.line diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index 783c2754a9d2..5a8b67adfc4b 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -68,6 +68,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, self.scope.enter_file(file_node.fullname()) self.accept(file_node) self.scope.leave() + del self.cur_mod_node + self.patches = [] def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None: """Refresh a stale target in fine-grained incremental mode.""" diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py index e15e780406b8..65642c473f14 100644 --- a/mypy/server/objgraph.py +++ b/mypy/server/objgraph.py @@ -28,8 +28,6 @@ 'indirection_detector', 'all_types', 'type_maps', - 'semantic_analyzer', # Semantic analyzer has stale caches - 'semantic_analyzer_pass3', # Semantic analyzer has stale caches } # Instances of these types can't have references to other objects From d97486e5e134a525e2f836bb9fd0e800456cbd04 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 10:48:10 +0000 Subject: [PATCH 13/23] Remove no longer needed attribute blacklist items --- mypy/server/objgraph.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py index 65642c473f14..2b95f311d097 100644 --- a/mypy/server/objgraph.py +++ b/mypy/server/objgraph.py @@ -25,9 +25,7 @@ '__dict__', # Mypy specific attribute blacklists - 'indirection_detector', 'all_types', - 'type_maps', } # Instances of these types can't have references to other objects From 8acdf48a54f0a2d6fff4c2204aaa703959f4f7b4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 11:27:26 +0000 Subject: [PATCH 14/23] Don't collect all types in fine-grained incremental tests The type map could contain stale references to AST nodes. --- mypy/build.py | 4 +++- mypy/server/objgraph.py | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ac8436ee10af..5226021494bf 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1992,7 +1992,9 @@ def finish_passes(self) -> None: return with self.wrap_context(): # Some tests want to look at the set of all types. - if manager.options.use_builtins_fixtures or manager.options.dump_deps: + options = manager.options + if ((options.use_builtins_fixtures and not options.fine_grained_incremental) or + manager.options.dump_deps): manager.all_types.update(self.type_map()) if self.options.incremental: diff --git a/mypy/server/objgraph.py b/mypy/server/objgraph.py index 2b95f311d097..b8ebd5a8e2f3 100644 --- a/mypy/server/objgraph.py +++ b/mypy/server/objgraph.py @@ -23,9 +23,6 @@ '__name__', '__class__', '__dict__', - - # Mypy specific attribute blacklists - 'all_types', } # Instances of these types can't have references to other objects From aa98711abf558e432c71b02475e68a6f83b42a1e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 11:30:43 +0000 Subject: [PATCH 15/23] Add another AST merge fixup Not sure if this is necessary but it might be. --- mypy/server/astmerge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 63bf6e640561..7b7d42c8eeb1 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -282,8 +282,8 @@ def process_type_info(self, info: Optional[TypeInfo]) -> None: # - declared_metaclass # - metaclass_type # - _promote - # - replaced info.defn.info = self.fixup(info) + info.replaced = self.fixup(info.replaced) if info.typeddict_type: self.fixup_type(info.typeddict_type) replace_nodes_in_symbol_table(info.names, self.replacements) From 6638d77a81d6f50d6b309d5df5eab0590c200c61 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 13:03:26 +0000 Subject: [PATCH 16/23] Fixes to AST merge; some refactoring --- mypy/server/astmerge.py | 52 +++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 7b7d42c8eeb1..fa41169dc66e 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -51,7 +51,7 @@ Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo, FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr, OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, CallExpr, - MDEF + Decorator, MDEF ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -179,10 +179,8 @@ def visit_class_def(self, node: ClassDef) -> None: super().visit_class_def(node) def process_base_func(self, node: FuncBase) -> None: - if node.type: - self.fixup_type(node.type) - if node.info: - node.info = self.fixup(node.info) + self.fixup_type(node.type) + node.info = self.fixup(node.info) if node.unanalyzed_type: # Unanalyzed types can have AST node references self.fixup_type(node.unanalyzed_type) @@ -193,10 +191,14 @@ def process_type_var_def(self, tv: TypeVarDef) -> None: self.fixup_type(tv.upper_bound) def visit_assignment_stmt(self, node: AssignmentStmt) -> None: - if node.type: - self.fixup_type(node.type) + self.fixup_type(node.type) super().visit_assignment_stmt(node) + def visit_decorator(self, node: Decorator) -> None: + super().visit_decorator(node) + node.func = self.fixup(node.func) + node.var = self.fixup(node.var) + # Expressions def visit_name_expr(self, node: NameExpr) -> None: @@ -211,6 +213,9 @@ def visit_member_expr(self, node: MemberExpr) -> None: def visit_ref_expr(self, node: RefExpr) -> None: if node.node is not None: node.node = self.fixup(node.node) + if isinstance(node.node, Var): + # The Var node may be an orphan and won't otherwise be processed. + fixup_var(node.node, self.replacements) def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None: super().visit_namedtuple_expr(node) @@ -230,13 +235,11 @@ def visit_newtype_expr(self, node: NewTypeExpr) -> None: if node.info: node.info = self.fixup(node.info) self.process_synthetic_type_info(node.info) - if node.old_type: - self.fixup_type(node.old_type) + self.fixup_type(node.old_type) super().visit_newtype_expr(node) def visit_lambda_expr(self, node: LambdaExpr) -> None: - if node.info: - node.info = self.fixup(node.info) + node.info = self.fixup(node.info) super().visit_lambda_expr(node) def visit_typeddict_expr(self, node: TypedDictExpr) -> None: @@ -257,10 +260,8 @@ def visit_type_alias_expr(self, node: TypeAliasExpr) -> None: # Others def visit_var(self, node: Var) -> None: - if node.info: - node.info = self.fixup(node.info) - if node.type: - self.fixup_type(node.type) + node.info = self.fixup(node.info) + self.fixup_type(node.type) super().visit_var(node) # Helpers @@ -272,8 +273,9 @@ def fixup(self, node: SN) -> SN: return cast(SN, new) return node - def fixup_type(self, typ: Type) -> None: - typ.accept(TypeReplaceVisitor(self.replacements)) + def fixup_type(self, typ: Optional[Type]) -> None: + if typ is not None: + typ.accept(TypeReplaceVisitor(self.replacements)) def process_type_info(self, info: Optional[TypeInfo]) -> None: if info is None: @@ -409,17 +411,17 @@ def replace_nodes_in_symbol_table(symbols: SymbolTable, node.node = new # TODO: Other node types if isinstance(node.node, Var): - if node.node.type: - node.node.type.accept(TypeReplaceVisitor(replacements)) - node.node.info = cast(TypeInfo, replacements.get(node.node.info, - node.node.info)) + fixup_var(node.node, replacements) else: # TODO: Other node types if isinstance(node.node, Var): - if node.node.type: - node.node.type.accept(TypeReplaceVisitor(replacements)) - node.node.info = cast(TypeInfo, replacements.get(node.node.info, - node.node.info)) + fixup_var(node.node, replacements) override = node.type_override if override: override.accept(TypeReplaceVisitor(replacements)) + + +def fixup_var(node: Var, replacements: Dict[SymbolNode, SymbolNode]) -> None: + if node.type: + node.type.accept(TypeReplaceVisitor(replacements)) + node.info = cast(TypeInfo, replacements.get(node.info, node.info)) From fe7672a6c71d02ebb57ba644d3ebc3d0139b1535 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 13:12:24 +0000 Subject: [PATCH 17/23] Remove debug code --- mypy/semanal.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 4f80258e89e1..5bdefe2e6817 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -991,9 +991,6 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> Optional[TypeInfo]: defn.name, items, types, default_items) node.node = info defn.info.replaced = info - # Mark that the original TypeInfo should no longer be used. - defn.info._fullname += '' - print('orig:', id(defn.info), 'replaced:', id(info)) defn.info = info defn.analyzed = NamedTupleExpr(info) defn.analyzed.line = defn.line From ee7afa95e5ce0db31d7dc4d708ce9476f4770f4b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 13:21:50 +0000 Subject: [PATCH 18/23] Fix merge test cases --- mypy/test/testmerge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index cc79958d5c15..6775a3b55893 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -8,7 +8,8 @@ from mypy.build import BuildManager, BuildSource, State from mypy.errors import Errors, CompileError from mypy.nodes import ( - Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, UNBOUND_IMPORTED + Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, TypeVarExpr, + UNBOUND_IMPORTED ) from mypy.options import Options from mypy.server.astmerge import merge_asts @@ -86,6 +87,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: a.extend(self.dump(manager, kind)) for expr in old_subexpr: + if isinstance(expr, TypeVarExpr): + # These are merged so we can't perform the check. + continue # Verify that old AST nodes are removed from the expression type map. assert expr not in new_types From 7cb8a90ef4113b66c9648ea16234c1dd5760e8e7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 13:32:32 +0000 Subject: [PATCH 19/23] Fix type check --- mypy/server/astmerge.py | 3 ++- mypy/types.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index fa41169dc66e..bb50d727a5e8 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -229,7 +229,8 @@ def visit_super_expr(self, node: SuperExpr) -> None: def visit_call_expr(self, node: CallExpr) -> None: super().visit_call_expr(node) - node.analyzed = self.fixup(node.analyzed) + if isinstance(node.analyzed, SymbolNode): + node.analyzed = self.fixup(node.analyzed) def visit_newtype_expr(self, node: NewTypeExpr) -> None: if node.info: diff --git a/mypy/types.py b/mypy/types.py index 226d7e998209..66a4d4da7ccd 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1503,6 +1503,11 @@ def visit_type_type(self, t: TypeType) -> T: def visit_forwardref_type(self, t: ForwardRef) -> T: raise RuntimeError('Internal error: unresolved forward reference') + def visit_type_list(self, t: TypeList) -> T: + # TODO: Do we need to implement this in more visitors? TypeList objects can + # exist as components of UnboundTypes. + raise self._notimplemented_helper('type_list') + class SyntheticTypeVisitor(TypeVisitor[T]): """A TypeVisitor that also knows how to visit synthetic AST constructs. From 9757374968810a5d87235ec9d1fbc6c96ce8544b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 13:48:54 +0000 Subject: [PATCH 20/23] Remove redundant fixups The relevant nodes won't be exposed externally. --- mypy/server/astmerge.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index bb50d727a5e8..414af8aac452 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -51,7 +51,7 @@ Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo, FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr, OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, CallExpr, - Decorator, MDEF + MDEF ) from mypy.traverser import TraverserVisitor from mypy.types import ( @@ -194,11 +194,6 @@ def visit_assignment_stmt(self, node: AssignmentStmt) -> None: self.fixup_type(node.type) super().visit_assignment_stmt(node) - def visit_decorator(self, node: Decorator) -> None: - super().visit_decorator(node) - node.func = self.fixup(node.func) - node.var = self.fixup(node.var) - # Expressions def visit_name_expr(self, node: NameExpr) -> None: From 992317803a00040c3de53d2756fa285c3efd95de Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 14:03:06 +0000 Subject: [PATCH 21/23] Merge additional TypeInfo attributes; minor refactoring --- mypy/nodes.py | 15 +++++++++++---- mypy/server/astmerge.py | 24 +++++++++--------------- test-data/unit/merge.test | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index c82d578050e6..fdb29dd25fa7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2206,11 +2206,18 @@ def type_str(typ: 'mypy.types.Type') -> str: if isinstance(node, Var) and node.type: description += ' ({})'.format(type_str(node.type)) names.append(description) + items = [ + 'Name({})'.format(self.fullname()), + base, + mro, + ('Names', names), + ] + if self.declared_metaclass: + items.append('DeclaredMetaclass({})'.format(type_str(self.declared_metaclass))) + if self.metaclass_type: + items.append('MetaclassType({})'.format(type_str(self.metaclass_type))) return mypy.strconv.dump_tagged( - ['Name({})'.format(self.fullname()), - base, - mro, - ('Names', names)], + items, head, str_conv=str_conv) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 414af8aac452..30f96fdab7f3 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -276,21 +276,18 @@ def fixup_type(self, typ: Optional[Type]) -> None: def process_type_info(self, info: Optional[TypeInfo]) -> None: if info is None: return - # TODO: Additional things: - # - declared_metaclass - # - metaclass_type - # - _promote + self.fixup_type(info.declared_metaclass) + self.fixup_type(info.metaclass_type) + self.fixup_type(info._promote) + self.fixup_type(info.tuple_type) + self.fixup_type(info.typeddict_type) info.defn.info = self.fixup(info) info.replaced = self.fixup(info.replaced) - if info.typeddict_type: - self.fixup_type(info.typeddict_type) replace_nodes_in_symbol_table(info.names, self.replacements) for i, item in enumerate(info.mro): info.mro[i] = self.fixup(info.mro[i]) for i, base in enumerate(info.bases): self.fixup_type(info.bases[i]) - if info.tuple_type: - self.fixup_type(info.tuple_type) def process_synthetic_type_info(self, info: TypeInfo) -> None: # Synthetic types (types not created using a class statement) don't @@ -405,13 +402,10 @@ def replace_nodes_in_symbol_table(symbols: SymbolTable, new = replacements[node.node] new.__dict__ = node.node.__dict__ node.node = new - # TODO: Other node types - if isinstance(node.node, Var): - fixup_var(node.node, replacements) - else: - # TODO: Other node types - if isinstance(node.node, Var): - fixup_var(node.node, replacements) + if isinstance(node.node, Var): + # Handle them here just in case these aren't exposed through the AST. + # TODO: Is this necessary? + fixup_var(node.node, replacements) override = node.type_override if override: override.accept(TypeReplaceVisitor(replacements)) diff --git a/test-data/unit/merge.test b/test-data/unit/merge.test index 5d539ce29e7a..d755b83d43dd 100644 --- a/test-data/unit/merge.test +++ b/test-data/unit/merge.test @@ -1358,3 +1358,40 @@ __main__: target: : TypeInfo<1> g: FuncDef<2> + +[case testMetaclass_typeinfo] +import target +[file target.py] +class M(type): pass +class C(metaclass=M): + pass +[file target.py.next] +class M(type): pass +class C(metaclass=M): + pass # dummy change +[out] +TypeInfo<0>( + Name(target.C) + Bases(builtins.object<1>) + Mro(target.C<0>, builtins.object<1>) + Names() + DeclaredMetaclass(target.M<2>) + MetaclassType(target.M<2>)) +TypeInfo<2>( + Name(target.M) + Bases(builtins.type<3>) + Mro(target.M<2>, builtins.type<3>, builtins.object<1>) + Names()) +==> +TypeInfo<0>( + Name(target.C) + Bases(builtins.object<1>) + Mro(target.C<0>, builtins.object<1>) + Names() + DeclaredMetaclass(target.M<2>) + MetaclassType(target.M<2>)) +TypeInfo<2>( + Name(target.M) + Bases(builtins.type<3>) + Mro(target.M<2>, builtins.type<3>, builtins.object<1>) + Names()) From 008a9a80188425d0ec00042c1933a9f9f9d4ee79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Feb 2018 14:10:49 +0000 Subject: [PATCH 22/23] Fix test failure --- test-data/unit/semanal-typeinfo.test | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test-data/unit/semanal-typeinfo.test b/test-data/unit/semanal-typeinfo.test index 098ce0b114ad..f4b92457df09 100644 --- a/test-data/unit/semanal-typeinfo.test +++ b/test-data/unit/semanal-typeinfo.test @@ -68,12 +68,15 @@ TypeInfoMap( Name(__main__.c) Bases(__main__.i) Mro(__main__.c, __main__.i, builtins.object) - Names()) + Names() + MetaclassType(abc.ABCMeta)) __main__.i : TypeInfo( Name(__main__.i) Bases(builtins.object) Mro(__main__.i, builtins.object) - Names())) + Names() + DeclaredMetaclass(abc.ABCMeta) + MetaclassType(abc.ABCMeta))) [case testAttributeWithoutType] class A: From e76119711d45c9a66ff057e6797a80a5eac6d572 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Mar 2018 10:38:46 +0000 Subject: [PATCH 23/23] Fix typo --- mypy/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/types.py b/mypy/types.py index 66a4d4da7ccd..7f9bd662fe4d 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -245,7 +245,7 @@ class TypeList(Type): This is used for the arguments of a Callable type, i.e. for [arg, ...] in Callable[[arg, ...], ret]. This is not a real type but a syntactic AST construct. UnboundTypes can also have TypeList - types before they are processed into Callable type.s + types before they are processed into Callable types. """ items = None # type: List[Type]