Fine-grained: Fix AST merge issues

mypy/build.py

@@ -1992,7 +1992,9 @@ def finish_passes(self) -> None:
             return
         with self.wrap_context():
             # Some tests want to look at the set of all types.
-            if manager.options.use_builtins_fixtures or manager.options.dump_deps:
+            options = manager.options
+            if ((options.use_builtins_fixtures and not options.fine_grained_incremental) or
+                    manager.options.dump_deps):
                 manager.all_types.update(self.type_map())
 
             if self.options.incremental:

mypy/nodes.py

@@ -2206,11 +2206,18 @@ def type_str(typ: 'mypy.types.Type') -> str:
             if isinstance(node, Var) and node.type:
                 description += ' ({})'.format(type_str(node.type))
             names.append(description)
+        items = [
+            'Name({})'.format(self.fullname()),
+            base,
+            mro,
+            ('Names', names),
+        ]
+        if self.declared_metaclass:
+            items.append('DeclaredMetaclass({})'.format(type_str(self.declared_metaclass)))
+        if self.metaclass_type:
+            items.append('MetaclassType({})'.format(type_str(self.metaclass_type)))
         return mypy.strconv.dump_tagged(
-            ['Name({})'.format(self.fullname()),
-             base,
-             mro,
-             ('Names', names)],
+            items,
             head,
             str_conv=str_conv)
 

mypy/semanal.py

@@ -306,6 +306,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
 
             del self.options
             del self.patches
+            del self.cur_mod_node
+            del self.globals
 
     def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""
@@ -324,6 +326,7 @@ def refresh_top_level(self, file_node: MypyFile) -> None:
         self.recurse_into_functions = False
         for d in file_node.defs:
             self.accept(d)
+        del self.patches
 
     @contextmanager
     def file_context(self, file_node: MypyFile, fnam: str, options: Options,

mypy/semanal_pass3.py

@@ -68,6 +68,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
             self.scope.enter_file(file_node.fullname())
             self.accept(file_node)
             self.scope.leave()
+        del self.cur_mod_node
+        self.patches = []
 
     def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""

mypy/server/astmerge.py

@@ -50,13 +50,14 @@
 from mypy.nodes import (
     Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo,
     FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr,
-    OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, MDEF
+    OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, CallExpr,
+    MDEF
 )
 from mypy.traverser import TraverserVisitor
 from mypy.types import (
     Type, TypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType,
     TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
-    Overloaded, TypeVarDef
+    Overloaded, TypeVarDef, TypeList
 )
 from mypy.util import get_prefix
 
@@ -156,34 +157,41 @@ def visit_block(self, node: Block) -> None:
 
     def visit_func_def(self, node: FuncDef) -> None:
         node = self.fixup(node)
-        if node.type:
-            self.fixup_type(node.type)
-        if node.info:
-            node.info = self.fixup(node.info)
+        self.process_base_func(node)
         super().visit_func_def(node)
 
     def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
+        self.process_base_func(node)
         super().visit_overloaded_func_def(node)
 
     def visit_class_def(self, node: ClassDef) -> None:
         # TODO additional things?
         node.defs.body = self.replace_statements(node.defs.body)
         node.info = self.fixup(node.info)
+        info = node.info
         for tv in node.type_vars:
             self.process_type_var_def(tv)
-        self.process_type_info(node.info)
+        if info:
+            if info.is_named_tuple:
+                self.process_synthetic_type_info(info)
+            else:
+                self.process_type_info(info)
         super().visit_class_def(node)
 
+    def process_base_func(self, node: FuncBase) -> None:
+        self.fixup_type(node.type)
+        node.info = self.fixup(node.info)
+        if node.unanalyzed_type:
+            # Unanalyzed types can have AST node references
+            self.fixup_type(node.unanalyzed_type)
+
     def process_type_var_def(self, tv: TypeVarDef) -> None:
         for value in tv.values:
             self.fixup_type(value)
         self.fixup_type(tv.upper_bound)
 
     def visit_assignment_stmt(self, node: AssignmentStmt) -> None:
-        if node.type:
-            self.fixup_type(node.type)
+        self.fixup_type(node.type)
         super().visit_assignment_stmt(node)
 
     # Expressions
@@ -200,46 +208,56 @@ def visit_member_expr(self, node: MemberExpr) -> None:
     def visit_ref_expr(self, node: RefExpr) -> None:
         if node.node is not None:
             node.node = self.fixup(node.node)
+            if isinstance(node.node, Var):
+                # The Var node may be an orphan and won't otherwise be processed.
+                fixup_var(node.node, self.replacements)
 
     def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None:
         super().visit_namedtuple_expr(node)
         node.info = self.fixup(node.info)
-        self.process_type_info(node.info)
+        self.process_synthetic_type_info(node.info)
 
     def visit_super_expr(self, node: SuperExpr) -> None:
         super().visit_super_expr(node)
         if node.info is not None:
             node.info = self.fixup(node.info)
 
+    def visit_call_expr(self, node: CallExpr) -> None:
+        super().visit_call_expr(node)
+        if isinstance(node.analyzed, SymbolNode):
+            node.analyzed = self.fixup(node.analyzed)
+
     def visit_newtype_expr(self, node: NewTypeExpr) -> None:
         if node.info:
             node.info = self.fixup(node.info)
-            self.process_type_info(node.info)
-        if node.old_type:
-            self.fixup_type(node.old_type)
+            self.process_synthetic_type_info(node.info)
+        self.fixup_type(node.old_type)
         super().visit_newtype_expr(node)
 
     def visit_lambda_expr(self, node: LambdaExpr) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
+        node.info = self.fixup(node.info)
         super().visit_lambda_expr(node)
 
     def visit_typeddict_expr(self, node: TypedDictExpr) -> None:
-        node.info = self.fixup(node.info)
         super().visit_typeddict_expr(node)
+        node.info = self.fixup(node.info)
+        self.process_synthetic_type_info(node.info)
 
     def visit_enum_call_expr(self, node: EnumCallExpr) -> None:
         node.info = self.fixup(node.info)
-        self.process_type_info(node.info)
+        self.process_synthetic_type_info(node.info)
         super().visit_enum_call_expr(node)
 
+    def visit_type_alias_expr(self, node: TypeAliasExpr) -> None:
+        self.fixup_type(node.type)
+        self.fixup_type(node.fallback)
+        super().visit_type_alias_expr(node)
+
     # Others
 
     def visit_var(self, node: Var) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
-        if node.type:
-            self.fixup_type(node.type)
+        node.info = self.fixup(node.info)
+        self.fixup_type(node.type)
         super().visit_var(node)
 
     # Helpers
@@ -251,25 +269,34 @@ def fixup(self, node: SN) -> SN:
             return cast(SN, new)
         return node
 
-    def fixup_type(self, typ: Type) -> None:
-        typ.accept(TypeReplaceVisitor(self.replacements))
+    def fixup_type(self, typ: Optional[Type]) -> None:
+        if typ is not None:
+            typ.accept(TypeReplaceVisitor(self.replacements))
 
     def process_type_info(self, info: Optional[TypeInfo]) -> None:
         if info is None:
             return
-        # TODO: Additional things:
-        # - declared_metaclass
-        # - metaclass_type
-        # - _promote
-        # - typeddict_type
-        # - replaced
+        self.fixup_type(info.declared_metaclass)
+        self.fixup_type(info.metaclass_type)
+        self.fixup_type(info._promote)
+        self.fixup_type(info.tuple_type)
+        self.fixup_type(info.typeddict_type)
+        info.defn.info = self.fixup(info)
+        info.replaced = self.fixup(info.replaced)
         replace_nodes_in_symbol_table(info.names, self.replacements)
         for i, item in enumerate(info.mro):
             info.mro[i] = self.fixup(info.mro[i])
         for i, base in enumerate(info.bases):
             self.fixup_type(info.bases[i])
-        if info.tuple_type:
-            self.fixup_type(info.tuple_type)
+
+    def process_synthetic_type_info(self, info: TypeInfo) -> None:
+        # Synthetic types (types not created using a class statement) don't
+        # have bodies in the AST so we need to iterate over their symbol
+        # tables separately, unlike normal classes.
+        self.process_type_info(info)
+        for name, node in info.names.items():
+            if node.node:
+                node.node.accept(self)
 
     def replace_statements(self, nodes: List[Statement]) -> List[Statement]:
         result = []
@@ -345,6 +372,10 @@ def visit_unbound_type(self, typ: UnboundType) -> None:
         for arg in typ.args:
             arg.accept(self)
 
+    def visit_type_list(self, typ: TypeList) -> None:
+        for item in typ.items:
+            item.accept(self)
+
     def visit_uninhabited_type(self, typ: UninhabitedType) -> None:
         pass
 
@@ -371,15 +402,16 @@ def replace_nodes_in_symbol_table(symbols: SymbolTable,
                 new = replacements[node.node]
                 new.__dict__ = node.node.__dict__
                 node.node = new
-                # TODO: Other node types
-                if isinstance(node.node, Var) and node.node.type:
-                    node.node.type.accept(TypeReplaceVisitor(replacements))
-                    node.node.info = cast(TypeInfo, replacements.get(node.node.info,
-                                                                     node.node.info))
-            else:
-                # TODO: Other node types
-                if isinstance(node.node, Var) and node.node.type:
-                    node.node.type.accept(TypeReplaceVisitor(replacements))
+            if isinstance(node.node, Var):
+                # Handle them here just in case these aren't exposed through the AST.
+                # TODO: Is this necessary?
+                fixup_var(node.node, replacements)
         override = node.type_override
         if override:
             override.accept(TypeReplaceVisitor(replacements))
+
+
+def fixup_var(node: Var, replacements: Dict[SymbolNode, SymbolNode]) -> None:
+    if node.type:
+        node.type.accept(TypeReplaceVisitor(replacements))
+    node.info = cast(TypeInfo, replacements.get(node.info, node.info))

mypy/server/objgraph.py

@@ -23,13 +23,6 @@
     '__name__',
     '__class__',
     '__dict__',
-
-    # Mypy specific attribute blacklists
-    'indirection_detector',
-    'all_types',
-    'type_maps',
-    'semantic_analyzer',  # Semantic analyzer has stale caches
-    'semantic_analyzer_pass3',  # Semantic analyzer has stale caches
 }
 
 # Instances of these types can't have references to other objects

mypy/server/update.py

@@ -698,9 +698,11 @@ def replace_modules_with_new_variants(
     for id in new_modules:
         new_module = new_modules[id]
         if id in old_modules and new_module is not None:
-            merge_asts(old_modules[id], old_modules[id].names,
+            preserved_module = old_modules[id]
+            merge_asts(preserved_module, old_modules[id].names,
                        new_module, new_module.names)
-            manager.modules[id] = old_modules[id]
+            manager.modules[id] = preserved_module
+            graph[id].tree = preserved_module
 
 
 def propagate_changes_using_dependencies(

mypy/test/testmerge.py

@@ -8,7 +8,8 @@
 from mypy.build import BuildManager, BuildSource, State
 from mypy.errors import Errors, CompileError
 from mypy.nodes import (
-    Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, UNBOUND_IMPORTED
+    Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, TypeVarExpr,
+    UNBOUND_IMPORTED
 )
 from mypy.options import Options
 from mypy.server.astmerge import merge_asts
@@ -86,6 +87,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         a.extend(self.dump(manager, kind))
 
         for expr in old_subexpr:
+            if isinstance(expr, TypeVarExpr):
+                # These are merged so we can't perform the check.
+                continue
             # Verify that old AST nodes are removed from the expression type map.
             assert expr not in new_types
 

mypy/traverser.py

@@ -59,6 +59,8 @@ def visit_class_def(self, o: ClassDef) -> None:
         for base in o.base_type_exprs:
             base.accept(self)
         o.defs.accept(self)
+        if o.analyzed:
+            o.analyzed.accept(self)
 
     def visit_decorator(self, o: Decorator) -> None:
         o.func.accept(self)

mypy/types.py

@@ -242,9 +242,10 @@ def serialize(self) -> JsonDict:
 class TypeList(Type):
     """Information about argument types and names [...].
 
-    This is only used for the arguments of a Callable type, i.e. for
+    This is used for the arguments of a Callable type, i.e. for
     [arg, ...] in Callable[[arg, ...], ret]. This is not a real type
-    but a syntactic AST construct.
+    but a syntactic AST construct. UnboundTypes can also have TypeList
+    types before they are processed into Callable types.
     """
 
     items = None  # type: List[Type]
@@ -254,7 +255,6 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None:
         self.items = items
 
     def accept(self, visitor: 'TypeVisitor[T]') -> T:
-        assert isinstance(visitor, SyntheticTypeVisitor)
         return visitor.visit_type_list(self)
 
     def serialize(self) -> JsonDict:
@@ -1503,6 +1503,11 @@ def visit_type_type(self, t: TypeType) -> T:
     def visit_forwardref_type(self, t: ForwardRef) -> T:
         raise RuntimeError('Internal error: unresolved forward reference')
 
+    def visit_type_list(self, t: TypeList) -> T:
+        # TODO: Do we need to implement this in more visitors? TypeList objects can
+        #   exist as components of UnboundTypes.
+        raise self._notimplemented_helper('type_list')
+
 
 class SyntheticTypeVisitor(TypeVisitor[T]):
     """A TypeVisitor that also knows how to visit synthetic AST constructs.