diff --git a/mypy/build.py b/mypy/build.py index dfa227077f3e..1cd0c250b8a7 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -32,8 +32,11 @@ from mypy.nodes import (MypyFile, ImportBase, Import, ImportFrom, ImportAll) from mypy.semanal_pass1 import SemanticAnalyzerPass1 +from mypy.newsemanal.semanal_pass1 import ReachabilityAnalyzer from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches from mypy.semanal_pass3 import SemanticAnalyzerPass3 +from mypy.newsemanal.semanal import NewSemanticAnalyzer +from mypy.newsemanal.semanal_main import semantic_analysis_for_scc from mypy.checker import TypeChecker from mypy.indirection import TypeIndirectionVisitor from mypy.errors import Errors, CompileError, report_internal_error @@ -502,10 +505,21 @@ def __init__(self, data_dir: str, self.modules = {} # type: Dict[str, MypyFile] self.missing_modules = set() # type: Set[str] self.plugin = plugin - self.semantic_analyzer = SemanticAnalyzerPass2(self.modules, self.missing_modules, - self.errors, self.plugin) - self.semantic_analyzer_pass3 = SemanticAnalyzerPass3(self.modules, self.errors, - self.semantic_analyzer) + if options.new_semantic_analyzer: + # Set of namespaces (module or class) that are being populated during semantic + # analysis and may have missing definitions. + self.incomplete_namespaces = set() # type: Set[str] + self.new_semantic_analyzer = NewSemanticAnalyzer( + self.modules, + self.missing_modules, + self.incomplete_namespaces, + self.errors, + self.plugin) + else: + self.semantic_analyzer = SemanticAnalyzerPass2(self.modules, self.missing_modules, + self.errors, self.plugin) + self.semantic_analyzer_pass3 = SemanticAnalyzerPass3(self.modules, self.errors, + self.semantic_analyzer) self.all_types = {} # type: Dict[Expression, Type] # Enabled by export_types self.indirection_detector = TypeIndirectionVisitor() self.stale_modules = set() # type: Set[str] @@ -1721,21 +1735,44 @@ def parse_file(self) -> None: modules[self.id] = self.tree - # Do the first pass of semantic analysis: add top-level - # definitions in the file to the symbol table. We must do - # this before processing imports, since this may mark some - # import statements as unreachable. - first = SemanticAnalyzerPass1(manager.semantic_analyzer) - with self.wrap_context(): - first.visit_file(self.tree, self.xpath, self.id, self.options) + self.semantic_analysis_pass1() - # Initialize module symbol table, which was populated by the - # semantic analyzer. - # TODO: Why can't SemanticAnalyzerPass1 .analyze() do this? - self.tree.names = manager.semantic_analyzer.globals + if not self.options.new_semantic_analyzer: + # Initialize module symbol table, which was populated by the + # semantic analyzer. + # TODO: Why can't SemanticAnalyzerPass1 .analyze() do this? + self.tree.names = manager.semantic_analyzer.globals self.check_blockers() + def semantic_analysis_pass1(self) -> None: + """Perform pass 1 of semantic analysis, which happens immediately after parsing. + + This pass can't assume that any other modules have been processed yet. + """ + options = self.options + assert self.tree is not None + if options.new_semantic_analyzer: + # Do the first pass of semantic analysis: analyze the reachability + # of blocks and import statements. We must do this before + # processing imports, since this may mark some import statements as + # unreachable. + # + # TODO: Once we remove the old semantic analyzer, this no longer should + # be considered as a semantic analysis pass -- it's an independent + # pass. + analyzer = ReachabilityAnalyzer() + with self.wrap_context(): + analyzer.visit_file(self.tree, self.xpath, self.id, options) + else: + # Do the first pass of semantic analysis: add top-level + # definitions in the file to the symbol table. We must do + # this before processing imports, since this may mark some + # import statements as unreachable. + first = SemanticAnalyzerPass1(self.manager.semantic_analyzer) + with self.wrap_context(): + first.visit_file(self.tree, self.xpath, self.id, options) + def compute_dependencies(self) -> None: """Compute a module's dependencies after parsing it. @@ -2649,12 +2686,15 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No typing_mod = graph['typing'].tree assert typing_mod, "The typing module was not parsed" manager.semantic_analyzer.add_builtin_aliases(typing_mod) - for id in stale: - graph[id].semantic_analysis() - for id in stale: - graph[id].semantic_analysis_pass_three() - for id in stale: - graph[id].semantic_analysis_apply_patches() + if manager.options.new_semantic_analyzer: + semantic_analysis_for_scc(graph, scc) + else: + for id in stale: + graph[id].semantic_analysis() + for id in stale: + graph[id].semantic_analysis_pass_three() + for id in stale: + graph[id].semantic_analysis_apply_patches() for id in stale: graph[id].type_check_first_pass() more = True diff --git a/mypy/main.py b/mypy/main.py index 21cf68ec96ab..d9ff726cc5fd 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -582,6 +582,8 @@ def add_invertible_flag(flag: str, "the contents of SHADOW_FILE instead.") add_invertible_flag('--fast-exit', default=False, help=argparse.SUPPRESS, group=internals_group) + add_invertible_flag('--new-semantic-analyzer', default=False, help=argparse.SUPPRESS, + group=internals_group) error_group = parser.add_argument_group( title='Error reporting', diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py index f8b4e6dfa619..31d3de688ba3 100644 --- a/mypy/newsemanal/semanal.py +++ b/mypy/newsemanal/semanal.py @@ -1,4 +1,4 @@ -"""The semantic analyzer passes 1 and 2. +"""The new semantic analyzer (work in progress). Bind names to definitions and do various other simple consistency checks. For example, consider this program: @@ -13,23 +13,7 @@ module-level variable (Var) node. The second assignment would also be analyzed, and the type of 'y' marked as being inferred. -Semantic analysis is the first analysis pass after parsing, and it is -subdivided into three passes: - - * SemanticAnalyzerPass1 is defined in mypy.semanal_pass1. - - * SemanticAnalyzerPass2 is the second pass. It does the bulk of the work. - It assumes that dependent modules have been semantically analyzed, - up to the second pass, unless there is a import cycle. - - * SemanticAnalyzerPass3 is the third pass. It's in mypy.semanal_pass3. - Semantic analysis of types is implemented in module mypy.typeanal. - -TODO: Check if the third pass slows down type checking significantly. - We could probably get rid of it -- for example, we could collect all - analyzed types in a collection and check them without having to - traverse the entire AST. """ from contextlib import contextmanager @@ -84,8 +68,8 @@ DynamicClassDefContext ) from mypy.util import get_prefix, correct_relative_import, unmangle -from mypy.semanal_shared import SemanticAnalyzerInterface, set_callable_name from mypy.scope import Scope +from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface, set_callable_name from mypy.newsemanal.semanal_namedtuple import NamedTupleAnalyzer, NAMEDTUPLE_PROHIBITED_NAMES from mypy.newsemanal.semanal_typeddict import TypedDictAnalyzer from mypy.newsemanal.semanal_enum import EnumCallAnalyzer @@ -164,16 +148,14 @@ } # type: Final -class SemanticAnalyzerPass2(NodeVisitor[None], - SemanticAnalyzerInterface, - SemanticAnalyzerPluginInterface): +class NewSemanticAnalyzer(NodeVisitor[None], + SemanticAnalyzerInterface, + SemanticAnalyzerPluginInterface): """Semantically analyze parsed mypy files. The analyzer binds names and does various consistency checks for a parse tree. Note that type checking is performed as a separate pass. - - This is the second phase of semantic analysis. """ # Module name space @@ -221,12 +203,18 @@ class SemanticAnalyzerPass2(NodeVisitor[None], def __init__(self, modules: Dict[str, MypyFile], missing_modules: Set[str], + incomplete_namespaces: Set[str], errors: Errors, plugin: Plugin) -> None: """Construct semantic analyzer. - Use lib_path to search for modules, and report analysis errors - using the Errors instance. + We reuse the same semantic analyzer instance across multiple modules. + + Args: + modules: Global modules dictionary + incomplete_namespaces: Namespaces that are being populated during semantic analysis + (can contain modules and classes within the current SCC) + errors: Report analysis errors using this instance """ self.locals = [None] self.imports = set() @@ -240,6 +228,7 @@ def __init__(self, self.modules = modules self.msg = MessageBuilder(errors, modules) self.missing_modules = missing_modules + self.incomplete_namespaces = incomplete_namespaces self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] self.postponed_functions_stack = [] self.all_exports = set() # type: Set[str] @@ -319,6 +308,20 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, del self.cur_mod_node del self.globals + def prepare_file(self, file_node: MypyFile) -> None: + """Prepare a freshly parsed file for semantic analysis.""" + file_node.names = SymbolTable() + if 'builtins' in self.modules: + file_node.names['__builtins__'] = SymbolTableNode(GDEF, + self.modules['builtins']) + if file_node.fullname() == 'builtins': + # Add empty core definitions required for basic operation. These fill be completed + # later on. + cdef = ClassDef('object', Block([])) # Dummy ClassDef, will be replaced later + info = TypeInfo(SymbolTable(), cdef, 'builtins') + info._fullname = 'builtins.object' + file_node.names['object'] = SymbolTableNode(GDEF, info) + def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], patches: List[Tuple[int, Callable[[], None]]]) -> None: """Refresh a stale target in fine-grained incremental mode.""" @@ -333,6 +336,7 @@ def refresh_partial(self, node: Union[MypyFile, FuncDef, OverloadedFuncDef], def refresh_top_level(self, file_node: MypyFile) -> None: """Reanalyze a stale module top-level in fine-grained incremental mode.""" self.recurse_into_functions = False + self.deferred = False # Set to true if another analysis pass is needed for d in file_node.defs: self.accept(d) @@ -351,6 +355,16 @@ def file_context(self, file_node: MypyFile, fnam: str, options: Options, self._is_typeshed_stub_file = self.errors.is_typeshed_file(file_node.path) self.globals = file_node.names self.tvar_scope = TypeVarScope() + + self.named_tuple_analyzer = NamedTupleAnalyzer(options, self) + self.typed_dict_analyzer = TypedDictAnalyzer(options, self, self.msg) + self.enum_call_analyzer = EnumCallAnalyzer(options, self) + self.newtype_analyzer = NewTypeAnalyzer(options, self, self.msg) + + # Counter that keeps track of references to undefined things potentially caused by + # incomplete namespaces. + self.num_incomplete_refs = 0 + if active_type: scope.enter_class(active_type) self.enter_class(active_type.defn.info) @@ -795,6 +809,7 @@ def check_function_signature(self, fdef: FuncItem) -> None: self.fail('Type signature has too many arguments', fdef, blocker=True) def visit_class_def(self, defn: ClassDef) -> None: + self.setup_class_def_analysis(defn) with self.scope.class_scope(defn.info): with self.tvar_scope_frame(self.tvar_scope.class_frame()): self.analyze_class(defn) @@ -808,7 +823,6 @@ def analyze_class(self, defn: ClassDef) -> None: return if self.analyze_namedtuple_classdef(defn): return - self.setup_class_def_analysis(defn) self.analyze_base_classes(defn) defn.info.is_protocol = is_protocol self.analyze_metaclass(defn) @@ -1128,8 +1142,24 @@ def get_all_bases_tvars(self, defn: ClassDef, removed: List[int]) -> TypeVarList def setup_class_def_analysis(self, defn: ClassDef) -> None: """Prepare for the analysis of a class definition.""" if not defn.info: - defn.info = TypeInfo(SymbolTable(), defn, self.cur_mod_id) - defn.info._fullname = defn.info.name() + defn.fullname = self.qualified_name(defn.name) + # TODO: Nested classes + if self.is_module_scope() and self.qualified_name(defn.name) == 'builtins.object': + # Special case 'builtins.object'. A TypeInfo was already + # created for it before semantic analysis, but with a dummy + # ClassDef. Patch the real ClassDef object. + info = self.globals['object'].node + assert isinstance(info, TypeInfo) + defn.info = info + info.defn = defn + else: + info = TypeInfo(SymbolTable(), defn, self.cur_mod_id) + defn.info = info + if self.is_module_scope(): + info._fullname = self.qualified_name(defn.name) + self.globals[defn.name] = SymbolTableNode(GDEF, info) + else: + info._fullname = info.name() if self.is_func_scope() or self.type: kind = MDEF if self.is_nested_within_func_scope(): @@ -1170,6 +1200,8 @@ def analyze_base_classes(self, defn: ClassDef) -> None: info.fallback_to_any = True continue + assert base is not None # TODO: Handle None values + if isinstance(base, TupleType): if info.tuple_type: self.fail("Class has two incompatible bases derived from tuple", defn) @@ -1293,7 +1325,9 @@ def update_metaclass(self, defn: ClassDef) -> None: return defn.metaclass = metas.pop() - def expr_to_analyzed_type(self, expr: Expression, report_invalid_types: bool = True) -> Type: + def expr_to_analyzed_type(self, + expr: Expression, + report_invalid_types: bool = True) -> Optional[Type]: if isinstance(expr, CallExpr): expr.accept(self) info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope()) @@ -1551,6 +1585,11 @@ def visit_import_from(self, imp: ImportFrom) -> None: self.add_symbol(imported_id, symbol, imp) elif module and not missing: # Missing attribute. + if import_id in self.incomplete_namespaces: + # We don't know whether the name will be there, since the namespace + # is incomplete. Defer the current target. + self.deferred = True + return message = "Module '{}' has no attribute '{}'".format(import_id, id) extra = self.undefined_name_extra_info('{}.{}'.format(import_id, id)) if extra: @@ -1726,13 +1765,23 @@ def anal_type(self, t: Type, *, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, report_invalid_types: bool = True, - third_pass: bool = False) -> Type: + third_pass: bool = False) -> Optional[Type]: + """Semantically analyze a type. + + Return None only if some part of the type couldn't be bound *and* it referred + to an incomplete namespace. In case of other errors, report an error message + and return AnyType. + """ a = self.type_analyzer(tvar_scope=tvar_scope, allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, report_invalid_types=report_invalid_types, third_pass=third_pass) + prev_incomplete = self.num_incomplete_refs typ = t.accept(a) + if self.num_incomplete_refs != prev_incomplete: + # Something could not be bound yet. + return None self.add_type_alias_deps(a.aliases_used) return typ @@ -1901,7 +1950,9 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: if s.type: lvalue = s.lvalues[-1] allow_tuple_literal = isinstance(lvalue, TupleExpr) - s.type = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) + analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + s.type = analyzed if (self.type and self.type.is_protocol and isinstance(lvalue, NameExpr) and isinstance(s.rvalue, TempNode) and s.rvalue.no_rhs): if isinstance(lvalue.node, Var): @@ -2066,7 +2117,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: node.node.normalized = rvalue.node.normalized def analyze_lvalue(self, lval: Lvalue, nested: bool = False, - add_global: bool = False, explicit_type: bool = False, is_final: bool = False) -> None: """Analyze an lvalue or assignment target. @@ -2076,30 +2126,27 @@ def analyze_lvalue(self, lval: Lvalue, nested: bool = False, Args: lval: The target lvalue nested: If true, the lvalue is within a tuple or list lvalue expression - add_global: Add name to globals table only if this is true (used in first pass) explicit_type: Assignment has type annotation """ if isinstance(lval, NameExpr): - self.analyze_name_lvalue(lval, add_global, explicit_type, is_final) + self.analyze_name_lvalue(lval, explicit_type, is_final) elif isinstance(lval, MemberExpr): - if not add_global: - self.analyze_member_lvalue(lval, explicit_type, is_final) + self.analyze_member_lvalue(lval, explicit_type, is_final) if explicit_type and not self.is_self_member_ref(lval): self.fail('Type cannot be declared in assignment to non-self ' 'attribute', lval) elif isinstance(lval, IndexExpr): if explicit_type: self.fail('Unexpected type declaration', lval) - if not add_global: - lval.accept(self) + lval.accept(self) elif isinstance(lval, TupleExpr): items = lval.items if len(items) == 0 and isinstance(lval, TupleExpr): self.fail("can't assign to ()", lval) - self.analyze_tuple_or_list_lvalue(lval, add_global, explicit_type) + self.analyze_tuple_or_list_lvalue(lval, explicit_type) elif isinstance(lval, StarExpr): if nested: - self.analyze_lvalue(lval.expr, nested, add_global, explicit_type) + self.analyze_lvalue(lval.expr, nested, explicit_type) else: self.fail('Starred assignment target must be in a list or tuple', lval) else: @@ -2107,7 +2154,6 @@ def analyze_lvalue(self, lval: Lvalue, nested: bool = False, def analyze_name_lvalue(self, lval: NameExpr, - add_global: bool, explicit_type: bool, is_final: bool) -> None: """Analyze an lvalue that targets a name expression. @@ -2122,10 +2168,8 @@ def analyze_name_lvalue(self, # Top-level definitions within some statements (at least while) are # not handled in the first pass, so they have to be added now. - nested_global = (not self.is_func_scope() and - self.block_depth[-1] > 0 and - not self.type) - if (add_global or nested_global) and lval.name not in self.globals: + add_global = not self.is_func_scope() and not self.type + if add_global and lval.name not in self.globals: # Define new global name. v = self.make_name_lvalue_var(lval, GDEF, not explicit_type) self.globals[lval.name] = SymbolTableNode(GDEF, v) @@ -2249,7 +2293,6 @@ def make_name_lvalue_point_to_existing_def( self.check_lvalue_validity(lval.node, lval) def analyze_tuple_or_list_lvalue(self, lval: TupleExpr, - add_global: bool = False, explicit_type: bool = False) -> None: """Analyze an lvalue or assignment target that is a list or tuple.""" items = lval.items @@ -2261,8 +2304,7 @@ def analyze_tuple_or_list_lvalue(self, lval: TupleExpr, if len(star_exprs) == 1: star_exprs[0].valid = True for i in items: - self.analyze_lvalue(i, nested=True, add_global=add_global, - explicit_type=explicit_type) + self.analyze_lvalue(i, nested=True, explicit_type=explicit_type) def analyze_member_lvalue(self, lval: MemberExpr, explicit_type: bool, is_final: bool) -> None: """Analyze lvalue that is a member expression. @@ -2480,8 +2522,10 @@ def process_typevar_parameters(self, args: List[Expression], try: # We want to use our custom error message below, so we suppress # the default error message for invalid types here. - upper_bound = self.expr_to_analyzed_type(param_value, - report_invalid_types=False) + analyzed = self.expr_to_analyzed_type(param_value, + report_invalid_types=False) + assert analyzed is not None # TODO: Handle None values + upper_bound = analyzed if isinstance(upper_bound, AnyType) and upper_bound.is_from_error: self.fail("TypeVar 'bound' must be a type", param_value) # Note: we do not return 'None' here -- we want to continue @@ -2531,7 +2575,12 @@ def analyze_types(self, items: List[Expression]) -> List[Type]: result = [] # type: List[Type] for node in items: try: - result.append(self.anal_type(expr_to_unanalyzed_type(node))) + analyzed = self.anal_type(expr_to_unanalyzed_type(node)) + if analyzed is not None: + result.append(analyzed) + else: + # TODO: Is this the right thing to do? + result.append(AnyType(TypeOfAny.from_error)) except TypeTranslationError: self.fail('Type expected', node) result.append(AnyType(TypeOfAny.from_error)) @@ -2765,8 +2814,10 @@ def visit_for_stmt(self, s: ForStmt) -> None: if self.is_classvar(s.index_type): self.fail_invalid_classvar(s.index) allow_tuple_literal = isinstance(s.index, TupleExpr) - s.index_type = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal) - self.store_declared_types(s.index, s.index_type) + analyzed = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + self.store_declared_types(s.index, analyzed) + s.index_type = analyzed self.loop_depth += 1 self.visit_block(s.body) @@ -2792,14 +2843,13 @@ def visit_if_stmt(self, s: IfStmt) -> None: def visit_try_stmt(self, s: TryStmt) -> None: self.analyze_try_stmt(s, self) - def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor[None], - add_global: bool = False) -> None: + def analyze_try_stmt(self, s: TryStmt, visitor: NodeVisitor[None]) -> None: s.body.accept(visitor) for type, var, handler in zip(s.types, s.vars, s.handlers): if type: type.accept(visitor) if var: - self.analyze_lvalue(var, add_global=add_global) + self.analyze_lvalue(var) handler.accept(visitor) if s.else_body: s.else_body.accept(visitor) @@ -2840,9 +2890,11 @@ def visit_with_stmt(self, s: WithStmt) -> None: if self.is_classvar(t): self.fail_invalid_classvar(n) allow_tuple_literal = isinstance(n, TupleExpr) - t = self.anal_type(t, allow_tuple_literal=allow_tuple_literal) - new_types.append(t) - self.store_declared_types(n, t) + analyzed = self.anal_type(t, allow_tuple_literal=allow_tuple_literal) + if analyzed is not None: + # TODO: Deal with this better + new_types.append(analyzed) + self.store_declared_types(n, analyzed) # Reverse the logic above to correctly reassign target_type if new_types: @@ -3247,8 +3299,9 @@ def visit_index_expr(self, expr: IndexExpr) -> None: # We always allow unbound type variables in IndexExpr, since we # may be analysing a type alias definition rvalue. The error will be # reported elsewhere if it is not the case. - typearg = self.anal_type(typearg, allow_unbound_tvars=True) - types.append(typearg) + typearg2 = self.anal_type(typearg, allow_unbound_tvars=True) + assert typearg2 is not None # TODO: Deal with None return values + types.append(typearg2) expr.analyzed = TypeApplication(expr.base, types) expr.analyzed.line = expr.line # Types list, dict, set are not subscriptable, prohibit this if @@ -3288,7 +3341,9 @@ def visit_slice_expr(self, expr: SliceExpr) -> None: def visit_cast_expr(self, expr: CastExpr) -> None: expr.expr.accept(self) - expr.type = self.anal_type(expr.type) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed def visit_reveal_expr(self, expr: RevealExpr) -> None: if expr.kind == REVEAL_TYPE: @@ -3302,7 +3357,9 @@ def visit_reveal_expr(self, expr: RevealExpr) -> None: def visit_type_application(self, expr: TypeApplication) -> None: expr.expr.accept(self) for i in range(len(expr.types)): - expr.types[i] = self.anal_type(expr.types[i]) + analyzed = self.anal_type(expr.types[i]) + if analyzed is not None: + expr.types[i] = analyzed def visit_list_comprehension(self, expr: ListComprehension) -> None: expr.generator.accept(self) @@ -3363,7 +3420,9 @@ def visit_backquote_expr(self, expr: BackquoteExpr) -> None: expr.expr.accept(self) def visit__promote_expr(self, expr: PromoteExpr) -> None: - expr.type = self.anal_type(expr.type) + analyzed = self.anal_type(expr.type) + if analyzed is not None: + expr.type = analyzed def visit_yield_expr(self, expr: YieldExpr) -> None: if not self.is_func_scope(): @@ -3721,6 +3780,13 @@ def check_no_global(self, n: str, ctx: Context, self.name_already_defined(n, ctx, self.globals[n]) def name_not_defined(self, name: str, ctx: Context) -> None: + # TODO: Reference to another namespace + if self.cur_mod_id in self.incomplete_namespaces: + # Target namespace is incomplete, so it's possible that the name will be defined + # later on. Defer current target. + self.deferred = True + self.num_incomplete_refs += 1 + return message = "Name '{}' is not defined".format(name) extra = self.undefined_name_extra_info(name) if extra: diff --git a/mypy/newsemanal/semanal_enum.py b/mypy/newsemanal/semanal_enum.py index 89242136d078..06999b60cbfa 100644 --- a/mypy/newsemanal/semanal_enum.py +++ b/mypy/newsemanal/semanal_enum.py @@ -10,7 +10,7 @@ UnicodeExpr, TupleExpr, ListExpr, DictExpr, Var, SymbolTableNode, GDEF, MDEF, ARG_POS, EnumCallExpr ) -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface from mypy.options import Options diff --git a/mypy/newsemanal/semanal_main.py b/mypy/newsemanal/semanal_main.py new file mode 100644 index 000000000000..534523902dd6 --- /dev/null +++ b/mypy/newsemanal/semanal_main.py @@ -0,0 +1,79 @@ +"""Top-level logic for the new semantic analyzer.""" + +from typing import List + +from mypy.nodes import Node, SymbolTable + +MYPY = False +if MYPY: + from mypy.build import Graph, State + + +def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None: + # Assume reachability analysis has already been performed. + process_top_levels(graph, scc) + process_functions(graph, scc) + + +def process_top_levels(graph: 'Graph', scc: List[str]) -> None: + # Process top levels until everything has been bound. + # TODO: Limit the number of iterations + + # Initialize ASTs and symbol tables. + for id in scc: + state = graph[id] + assert state.tree is not None + state.manager.new_semantic_analyzer.prepare_file(state.tree) + + # Initially all namespaces in the SCC are incomplete (well they are empty). + state.manager.incomplete_namespaces.update(scc) + + worklist = scc[:] + while worklist: + deferred = [] # type: List[str] + while worklist: + next_id = worklist.pop() + deferred += semantic_analyze_target(next_id, graph[next_id]) + # Assume this namespace is ready. + # TODO: It could still be incomplete if some definitions couldn't be bound. + state.manager.incomplete_namespaces.discard(next_id) + worklist = deferred + + +def process_functions(graph: 'Graph', scc: List[str]) -> None: + # TODO: This doesn't quite work yet + # Process functions. + deferred = [] # type: List[str] + for id in scc: + tree = graph[id].tree + assert tree is not None + symtable = tree.names + targets = get_all_leaf_targets(symtable) + for target in targets: + deferred += semantic_analyze_target(target, graph[id]) + assert not deferred # There can't be cross-function forward refs + + +def get_all_leaf_targets(symtable: SymbolTable) -> List[str]: + # TODO: Implement + return [] + + +def semantic_analyze_target(target: str, state: 'State') -> List[str]: + # TODO: Support refreshing function targets (currently only works for module top levels) + tree = state.tree + assert tree is not None + analyzer = state.manager.new_semantic_analyzer + # TODO: Move initialization to somewhere else + analyzer.global_decls = [set()] + analyzer.nonlocal_decls = [set()] + analyzer.globals = tree.names + with analyzer.file_context(file_node=tree, + fnam=tree.path, + options=state.options, + active_type=None): + analyzer.refresh_partial(tree, []) + if analyzer.deferred: + return [target] + else: + return [] diff --git a/mypy/newsemanal/semanal_namedtuple.py b/mypy/newsemanal/semanal_namedtuple.py index 47370a0a8efd..5b339143bd23 100644 --- a/mypy/newsemanal/semanal_namedtuple.py +++ b/mypy/newsemanal/semanal_namedtuple.py @@ -8,7 +8,9 @@ from mypy.types import ( Type, TupleType, NoneTyp, AnyType, TypeOfAny, TypeVarType, TypeVarDef, CallableType, TypeType ) -from mypy.semanal_shared import SemanticAnalyzerInterface, set_callable_name, PRIORITY_FALLBACKS +from mypy.newsemanal.semanal_shared import ( + SemanticAnalyzerInterface, set_callable_name, PRIORITY_FALLBACKS +) from mypy.nodes import ( Var, EllipsisExpr, Argument, StrExpr, BytesExpr, UnicodeExpr, ExpressionStmt, NameExpr, AssignmentStmt, PassStmt, Decorator, FuncBase, ClassDef, Expression, RefExpr, TypeInfo, @@ -92,9 +94,12 @@ def check_namedtuple_classdef( # Append name and type in this case... name = stmt.lvalues[0].name items.append(name) - types.append(AnyType(TypeOfAny.unannotated) - if stmt.type is None - else self.api.anal_type(stmt.type)) + if stmt.type is None: + types.append(AnyType(TypeOfAny.unannotated)) + else: + analyzed = self.api.anal_type(stmt.type) + assert analyzed is not None # TODO: Handle None values + types.append(analyzed) # ...despite possible minor failures that allow further analyzis. if name.startswith('_'): self.fail('NamedTuple field name cannot start with an underscore: {}' @@ -278,7 +283,9 @@ def parse_namedtuple_fields_with_types(self, nodes: List[Expression], context: C type = expr_to_unanalyzed_type(type_node) except TypeTranslationError: return self.fail_namedtuple_arg('Invalid field type', type_node) - types.append(self.api.anal_type(type)) + analyzed = self.api.anal_type(type) + assert analyzed is not None # TODO: Handle None values + types.append(analyzed) else: return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item) return items, types, [], True diff --git a/mypy/newsemanal/semanal_newtype.py b/mypy/newsemanal/semanal_newtype.py index 0846bc67ae4f..c4734026dcd1 100644 --- a/mypy/newsemanal/semanal_newtype.py +++ b/mypy/newsemanal/semanal_newtype.py @@ -10,7 +10,7 @@ AssignmentStmt, NewTypeExpr, CallExpr, NameExpr, RefExpr, Context, StrExpr, BytesExpr, UnicodeExpr, Block, FuncDef, Argument, TypeInfo, Var, SymbolTableNode, GDEF, MDEF, ARG_POS ) -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface from mypy.options import Options from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.newsemanal.typeanal import check_for_explicit_any, has_any_from_unimported_type diff --git a/mypy/newsemanal/semanal_pass1.py b/mypy/newsemanal/semanal_pass1.py index 7ef4effeafd3..fedf782a3ea9 100644 --- a/mypy/newsemanal/semanal_pass1.py +++ b/mypy/newsemanal/semanal_pass1.py @@ -1,390 +1,80 @@ -"""The semantic analyzer pass 1. +"""Block/import reachability analysis.""" -This sets up externally visible names defined in a module but doesn't -follow imports and mostly ignores local definitions. It helps enable -(some) cyclic references between modules, such as module 'a' that -imports module 'b' and used names defined in 'b' *and* vice versa. The -first pass can be performed before dependent modules have been -processed. - -Since this pass can't assume that other modules have been processed, -this pass cannot detect certain definitions that can only be recognized -in later passes. Examples of these include TypeVar and NamedTuple -definitions, as these look like regular assignments until we are able to -bind names, which only happens in pass 2. - -This pass also infers the reachability of certain if statements, such as -those with platform checks. This lets us filter out unreachable imports -at an early stage. -""" - -from typing import List, Tuple - -from mypy import state from mypy.nodes import ( - MypyFile, SymbolTable, SymbolTableNode, Var, Block, AssignmentStmt, FuncDef, Decorator, - ClassDef, TypeInfo, ImportFrom, Import, ImportAll, IfStmt, WhileStmt, ForStmt, WithStmt, - TryStmt, OverloadedFuncDef, Lvalue, Context, ImportedName, LDEF, GDEF, MDEF, UNBOUND_IMPORTED, - implicit_module_attrs, AssertStmt, + MypyFile, AssertStmt, IfStmt, Block, AssignmentStmt, ExpressionStmt, ReturnStmt, ForStmt ) -from mypy.types import Type, UnboundType, UnionType, AnyType, TypeOfAny, NoneTyp, CallableType -from mypy.newsemanal.semanal import SemanticAnalyzerPass2 -from mypy.reachability import infer_reachability_of_if_statement, assert_will_always_fail -from mypy.semanal_shared import create_indirect_imported_name +from mypy.traverser import TraverserVisitor from mypy.options import Options -from mypy.sametypes import is_same_type -from mypy.visitor import NodeVisitor -from mypy.renaming import VariableRenameVisitor - - -class SemanticAnalyzerPass1(NodeVisitor[None]): - """First phase of semantic analysis. - - See docstring of 'visit_file()' below and the module docstring for a - description of what this does. - """ - - def __init__(self, sem: SemanticAnalyzerPass2) -> None: - self.sem = sem - - def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) -> None: - """Perform the first analysis pass. - - Populate module global table. Resolve the full names of - definitions not nested within functions and construct type - info structures, but do not resolve inter-definition - references such as base classes. +from mypy.reachability import infer_reachability_of_if_statement, assert_will_always_fail - Also add implicit definitions such as __name__. - In this phase we don't resolve imports. For 'from ... import', - we generate dummy symbol table nodes for the imported names, - and these will get resolved in later phases of semantic - analysis. - """ - if options.allow_redefinition: - # Perform renaming across the AST to allow variable redefinitions - file.accept(VariableRenameVisitor()) - sem = self.sem - self.sem.options = options # Needed because we sometimes call into it - self.pyversion = options.python_version - self.platform = options.platform - sem.cur_mod_id = mod_id - sem.cur_mod_node = file - sem.errors.set_file(fnam, mod_id, scope=sem.scope) - sem.globals = SymbolTable() - sem.global_decls = [set()] - sem.nonlocal_decls = [set()] - sem.block_depth = [0] +class ReachabilityAnalyzer(TraverserVisitor): + """Analyze reachability of blocks and imports. - sem.scope.enter_file(mod_id) + This determines static reachability of blocks and imports due to version and + platform checks, among others. - defs = file.defs + The main entry point is 'visit_file'. - with state.strict_optional_set(options.strict_optional): - # Add implicit definitions of module '__name__' etc. - for name, t in implicit_module_attrs.items(): - # unicode docstrings should be accepted in Python 2 - if name == '__doc__': - if self.pyversion >= (3, 0): - typ = UnboundType('__builtins__.str') # type: Type - else: - typ = UnionType([UnboundType('__builtins__.str'), - UnboundType('__builtins__.unicode')]) - else: - assert t is not None, 'type should be specified for {}'.format(name) - typ = UnboundType(t) - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v) + Reachability of imports needs to be determined very early in the build since + this affects which modules will ultimately be processed. - for i, d in enumerate(defs): - d.accept(self) - if isinstance(d, AssertStmt) and assert_will_always_fail(d, options): - # We've encountered an assert that's always false, - # e.g. assert sys.platform == 'lol'. Truncate the - # list of statements. This mutates file.defs too. - del defs[i + 1:] - break + Consider this example: - # Add implicit definition of literals/keywords to builtins, as we - # cannot define a variable with them explicitly. - if mod_id == 'builtins': - literal_types = [ - ('None', NoneTyp()), - # reveal_type is a mypy-only function that gives an error with - # the type of its arg. - ('reveal_type', AnyType(TypeOfAny.special_form)), - # reveal_locals is a mypy-only function that gives an error with the types of - # locals - ('reveal_locals', AnyType(TypeOfAny.special_form)), - ] # type: List[Tuple[str, Type]] + import sys - # TODO(ddfisher): This guard is only needed because mypy defines - # fake builtins for its tests which often don't define bool. If - # mypy is fast enough that we no longer need those, this - # conditional check should be removed. - if 'bool' in self.sem.globals: - bool_type = self.sem.named_type('bool') - literal_types.extend([ - ('True', bool_type), - ('False', bool_type), - ('__debug__', bool_type), - ]) - else: - # We are running tests without 'bool' in builtins. - # TODO: Find a permanent solution to this problem. - # Maybe add 'bool' to all fixtures? - literal_types.append(('True', AnyType(TypeOfAny.special_form))) + def do_stuff(): + # type: () -> None: + if sys.python_version < (3,): + import xyz # Only available in Python 2 + xyz.whatever() + ... - for name, typ in literal_types: - v = Var(name, typ) - v._fullname = self.sem.qualified_name(name) - self.sem.globals[name] = SymbolTableNode(GDEF, v) + The block containing 'import xyz' is unreachable in Python 3 mode. The import + shouldn't be processed in Python 3 mode, even if the module happens to exist. + """ - del self.sem.options + def visit_file(self, file: MypyFile, fnam: str, mod_id: str, options: Options) -> None: + self.pyversion = options.python_version + self.platform = options.platform + self.cur_mod_id = mod_id + self.cur_mod_node = file + self.options = options + + for i, defn in enumerate(file.defs): + defn.accept(self) + if isinstance(defn, AssertStmt) and assert_will_always_fail(defn, options): + # We've encountered an assert that's always false, + # e.g. assert sys.platform == 'lol'. Truncate the + # list of statements. This mutates file.defs too. + del file.defs[i + 1:] + break - sem.scope.leave() + def visit_if_stmt(self, s: IfStmt) -> None: + infer_reachability_of_if_statement(s, self.options) + for node in s.body: + node.accept(self) + if s.else_body: + s.else_body.accept(self) def visit_block(self, b: Block) -> None: if b.is_unreachable: return - self.sem.block_depth[-1] += 1 - for node in b.body: - node.accept(self) - self.sem.block_depth[-1] -= 1 - - def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - if self.sem.is_module_scope(): - for lval in s.lvalues: - self.analyze_lvalue(lval, explicit_type=s.type is not None) - - def visit_func_def(self, func: FuncDef, decorated: bool = False) -> None: - """Process a func def. - - decorated is true if we are processing a func def in a - Decorator that needs a _fullname and to have its body analyzed but - does not need to be added to the symbol table. - """ - sem = self.sem - if sem.type is not None: - # Don't process methods during pass 1. - return - func.is_conditional = sem.block_depth[-1] > 0 - func._fullname = sem.qualified_name(func.name()) - at_module = sem.is_module_scope() and not decorated - if (at_module and func.name() == '__getattr__' and - self.sem.cur_mod_node.is_package_init_file() and self.sem.cur_mod_node.is_stub): - if isinstance(func.type, CallableType): - ret = func.type.ret_type - if isinstance(ret, UnboundType) and not ret.args: - sym = self.sem.lookup_qualified(ret.name, func, suppress_errors=True) - # We only interpret a package as partial if the __getattr__ return type - # is either types.ModuleType of Any. - if sym and sym.node and sym.node.fullname() in ('types.ModuleType', - 'typing.Any'): - self.sem.cur_mod_node.is_partial_stub_package = True - if at_module and func.name() in sem.globals: - # Already defined in this module. - original_sym = sem.globals[func.name()] - if (original_sym.kind == UNBOUND_IMPORTED or - isinstance(original_sym.node, ImportedName)): - # Ah this is an imported name. We can't resolve them now, so we'll postpone - # this until the main phase of semantic analysis. - return - if not sem.set_original_def(original_sym.node, func): - # Report error. - sem.check_no_global(func.name(), func) - else: - if at_module: - sem.globals[func.name()] = SymbolTableNode(GDEF, func) - # Also analyze the function body (needed in case there are unreachable - # conditional imports). - sem.function_stack.append(func) - sem.scope.enter_function(func) - sem.enter() - func.body.accept(self) - sem.leave() - sem.scope.leave() - sem.function_stack.pop() - - def visit_overloaded_func_def(self, func: OverloadedFuncDef) -> None: - if self.sem.type is not None: - # Don't process methods during pass 1. - return - kind = self.kind_by_scope() - if kind == GDEF: - self.sem.check_no_global(func.name(), func, True) - func._fullname = self.sem.qualified_name(func.name()) - if kind == GDEF: - self.sem.globals[func.name()] = SymbolTableNode(kind, func) - if func.impl: - impl = func.impl - # Also analyze the function body (in case there are conditional imports). - sem = self.sem + super().visit_block(b) - if isinstance(impl, FuncDef): - sem.function_stack.append(impl) - sem.scope.enter_function(func) - sem.enter() - impl.body.accept(self) - elif isinstance(impl, Decorator): - sem.function_stack.append(impl.func) - sem.scope.enter_function(func) - sem.enter() - impl.func.body.accept(self) - else: - assert False, "Implementation of an overload needs to be FuncDef or Decorator" - sem.leave() - sem.scope.leave() - sem.function_stack.pop() + # The remaining methods are an optimization: don't visit nested expressions + # of common statements, since they can have no effect. - def visit_class_def(self, cdef: ClassDef) -> None: - kind = self.kind_by_scope() - if kind == LDEF: - return - elif kind == GDEF: - self.sem.check_no_global(cdef.name, cdef) - cdef.fullname = self.sem.qualified_name(cdef.name) - info = TypeInfo(SymbolTable(), cdef, self.sem.cur_mod_id) - info.set_line(cdef.line, cdef.column) - cdef.info = info - if kind == GDEF: - self.sem.globals[cdef.name] = SymbolTableNode(kind, info) - self.process_nested_classes(cdef) - - def process_nested_classes(self, outer_def: ClassDef) -> None: - self.sem.enter_class(outer_def.info) - for node in outer_def.defs.body: - if isinstance(node, ClassDef): - node.info = TypeInfo(SymbolTable(), node, self.sem.cur_mod_id) - if outer_def.fullname: - node.info._fullname = outer_def.fullname + '.' + node.info.name() - else: - node.info._fullname = node.info.name() - node.fullname = node.info._fullname - symbol = SymbolTableNode(MDEF, node.info) - outer_def.info.names[node.name] = symbol - self.process_nested_classes(node) - elif isinstance(node, (ImportFrom, Import, ImportAll, IfStmt)): - node.accept(self) - self.sem.leave_class() - - def visit_import_from(self, node: ImportFrom) -> None: - # We can't bind module names during the first pass, as the target module might be - # unprocessed. However, we add dummy unbound imported names to the symbol table so - # that we at least know that the name refers to a module. - at_module = self.sem.is_module_scope() - node.is_top_level = at_module - if not at_module: - return - for name, as_name in node.names: - imported_name = as_name or name - if imported_name not in self.sem.globals: - sym = create_indirect_imported_name(self.sem.cur_mod_node, - node.id, - node.relative, - name) - if sym: - self.add_symbol(imported_name, sym, context=node) - - def visit_import(self, node: Import) -> None: - node.is_top_level = self.sem.is_module_scope() - # This is similar to visit_import_from -- see the comment there. - if not self.sem.is_module_scope(): - return - for id, as_id in node.ids: - imported_id = as_id or id - # For 'import a.b.c' we create symbol 'a'. - imported_id = imported_id.split('.')[0] - if imported_id not in self.sem.globals: - self.add_symbol(imported_id, SymbolTableNode(UNBOUND_IMPORTED, None), node) + def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + pass - def visit_import_all(self, node: ImportAll) -> None: - node.is_top_level = self.sem.is_module_scope() + def visit_expression_stmt(self, s: ExpressionStmt) -> None: + pass - def visit_while_stmt(self, s: WhileStmt) -> None: - if self.sem.is_module_scope(): - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) + def visit_return_stmt(self, s: ReturnStmt) -> None: + pass def visit_for_stmt(self, s: ForStmt) -> None: - if self.sem.is_module_scope(): - self.analyze_lvalue(s.index, explicit_type=s.index_type is not None) - s.body.accept(self) - if s.else_body: - s.else_body.accept(self) - - def visit_with_stmt(self, s: WithStmt) -> None: - if self.sem.is_module_scope(): - for n in s.target: - if n: - self.analyze_lvalue(n, explicit_type=s.target_type is not None) - s.body.accept(self) - - def visit_decorator(self, d: Decorator) -> None: - if self.sem.type is not None: - # Don't process methods during pass 1. - return - d.var._fullname = self.sem.qualified_name(d.var.name()) - self.add_symbol(d.var.name(), SymbolTableNode(self.kind_by_scope(), d), d) - self.visit_func_def(d.func, decorated=True) - - def visit_if_stmt(self, s: IfStmt) -> None: - infer_reachability_of_if_statement(s, self.sem.options) - for node in s.body: - node.accept(self) - if s.else_body: + s.body.accept(self) + if s.else_body is not None: s.else_body.accept(self) - - def visit_try_stmt(self, s: TryStmt) -> None: - if self.sem.is_module_scope(): - self.sem.analyze_try_stmt(s, self, add_global=self.sem.is_module_scope()) - - def analyze_lvalue(self, lvalue: Lvalue, explicit_type: bool = False) -> None: - self.sem.analyze_lvalue(lvalue, add_global=self.sem.is_module_scope(), - explicit_type=explicit_type) - - def kind_by_scope(self) -> int: - if self.sem.is_module_scope(): - return GDEF - elif self.sem.is_class_scope(): - return MDEF - elif self.sem.is_func_scope(): - return LDEF - else: - assert False, "Couldn't determine scope" - - def add_symbol(self, name: str, node: SymbolTableNode, - context: Context) -> None: - # NOTE: This is closely related to SemanticAnalyzerPass2.add_symbol. Since both methods - # will be called on top-level definitions, they need to co-operate. If you change - # this, you may have to change the other method as well. - if self.sem.is_func_scope(): - assert self.sem.locals[-1] is not None - if name in self.sem.locals[-1]: - # Flag redefinition unless this is a reimport of a module. - if not (isinstance(node.node, MypyFile) and - self.sem.locals[-1][name].node == node.node): - self.sem.name_already_defined(name, context, self.sem.locals[-1][name]) - return - self.sem.locals[-1][name] = node - else: - assert self.sem.type is None # Pass 1 doesn't look inside classes - existing = self.sem.globals.get(name) - if (existing - and (not isinstance(node.node, MypyFile) or existing.node != node.node) - and existing.kind != UNBOUND_IMPORTED - and not isinstance(existing.node, ImportedName)): - # Modules can be imported multiple times to support import - # of multiple submodules of a package (e.g. a.x and a.y). - ok = False - # Only report an error if the symbol collision provides a different type. - if existing.type and node.type and is_same_type(existing.type, node.type): - ok = True - if not ok: - self.sem.name_already_defined(name, context, existing) - return - elif not existing: - self.sem.globals[name] = node diff --git a/mypy/newsemanal/semanal_pass3.py b/mypy/newsemanal/semanal_pass3.py index 7b05c5516d95..88bb0221beba 100644 --- a/mypy/newsemanal/semanal_pass3.py +++ b/mypy/newsemanal/semanal_pass3.py @@ -29,12 +29,12 @@ from mypy.traverser import TraverserVisitor from mypy.newsemanal.typeanal import TypeAnalyserPass3, collect_any_types from mypy.typevars import has_no_typevars -from mypy.semanal_shared import PRIORITY_FORWARD_REF, PRIORITY_TYPEVAR_VALUES -from mypy.newsemanal.semanal import SemanticAnalyzerPass2 +from mypy.newsemanal.semanal_shared import PRIORITY_FORWARD_REF, PRIORITY_TYPEVAR_VALUES +from mypy.newsemanal.semanal import NewSemanticAnalyzer from mypy.subtypes import is_subtype from mypy.sametypes import is_same_type from mypy.scope import Scope -from mypy.semanal_shared import SemanticAnalyzerCoreInterface +from mypy.newsemanal.semanal_shared import SemanticAnalyzerCoreInterface class SemanticAnalyzerPass3(TraverserVisitor, SemanticAnalyzerCoreInterface): @@ -45,7 +45,7 @@ class SemanticAnalyzerPass3(TraverserVisitor, SemanticAnalyzerCoreInterface): """ def __init__(self, modules: Dict[str, MypyFile], errors: Errors, - sem: SemanticAnalyzerPass2) -> None: + sem: NewSemanticAnalyzer) -> None: self.modules = modules self.errors = errors self.sem = sem diff --git a/mypy/newsemanal/semanal_shared.py b/mypy/newsemanal/semanal_shared.py new file mode 100644 index 000000000000..67d25893cedb --- /dev/null +++ b/mypy/newsemanal/semanal_shared.py @@ -0,0 +1,158 @@ +"""Shared definitions used by different parts of semantic analysis.""" + +from abc import abstractmethod, abstractproperty +from typing import Optional, List, Callable +from mypy_extensions import trait + +from mypy.nodes import ( + Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF +) +from mypy.util import correct_relative_import +from mypy.types import Type, FunctionLike, Instance +from mypy.tvar_scope import TypeVarScope + +MYPY = False +if False: + from typing_extensions import Final + +# Priorities for ordering of patches within the final "patch" phase of semantic analysis +# (after pass 3): + +# Fix forward references (needs to happen first) +PRIORITY_FORWARD_REF = 0 # type: Final +# Fix fallbacks (does joins) +PRIORITY_FALLBACKS = 1 # type: Final +# Checks type var values (does subtype checks) +PRIORITY_TYPEVAR_VALUES = 2 # type: Final + + +@trait +class SemanticAnalyzerCoreInterface: + """A core abstract interface to generic semantic analyzer functionality. + + This is implemented by both semantic analyzer passes 2 and 3. + """ + + @abstractmethod + def lookup_qualified(self, name: str, ctx: Context, + suppress_errors: bool = False) -> Optional[SymbolTableNode]: + raise NotImplementedError + + @abstractmethod + def lookup_fully_qualified(self, name: str) -> SymbolTableNode: + raise NotImplementedError + + @abstractmethod + def fail(self, msg: str, ctx: Context, serious: bool = False, *, + blocker: bool = False) -> None: + raise NotImplementedError + + @abstractmethod + def note(self, msg: str, ctx: Context) -> None: + raise NotImplementedError + + @abstractmethod + def dereference_module_cross_ref( + self, node: Optional[SymbolTableNode]) -> Optional[SymbolTableNode]: + raise NotImplementedError + + +@trait +class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): + """A limited abstract interface to some generic semantic analyzer pass 2 functionality. + + We use this interface for various reasons: + + * Looser coupling + * Cleaner import graph + * Less need to pass around callback functions + """ + + @abstractmethod + def lookup(self, name: str, ctx: Context, + suppress_errors: bool = False) -> Optional[SymbolTableNode]: + raise NotImplementedError + + @abstractmethod + def named_type(self, qualified_name: str, args: Optional[List[Type]] = None) -> Instance: + raise NotImplementedError + + @abstractmethod + def named_type_or_none(self, qualified_name: str, + args: Optional[List[Type]] = None) -> Optional[Instance]: + raise NotImplementedError + + @abstractmethod + def accept(self, node: Node) -> None: + raise NotImplementedError + + @abstractmethod + def anal_type(self, t: Type, *, + tvar_scope: Optional[TypeVarScope] = None, + allow_tuple_literal: bool = False, + allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, + third_pass: bool = False) -> Optional[Type]: + raise NotImplementedError + + @abstractmethod + def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance) -> TypeInfo: + raise NotImplementedError + + @abstractmethod + def schedule_patch(self, priority: int, fn: Callable[[], None]) -> None: + raise NotImplementedError + + @abstractmethod + def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> None: + """Add node to global symbol table (or to nearest class if there is one).""" + raise NotImplementedError + + @abstractmethod + def parse_bool(self, expr: Expression) -> Optional[bool]: + raise NotImplementedError + + @abstractmethod + def qualified_name(self, n: str) -> str: + raise NotImplementedError + + @abstractproperty + def is_typeshed_stub_file(self) -> bool: + raise NotImplementedError + + +def create_indirect_imported_name(file_node: MypyFile, + module: str, + relative: int, + imported_name: str) -> Optional[SymbolTableNode]: + """Create symbol table entry for a name imported from another module. + + These entries act as indirect references. + """ + target_module, ok = correct_relative_import( + file_node.fullname(), + relative, + module, + file_node.is_package_init_file()) + if not ok: + return None + target_name = '%s.%s' % (target_module, imported_name) + link = ImportedName(target_name) + # Use GDEF since this refers to a module-level definition. + return SymbolTableNode(GDEF, link) + + +def set_callable_name(sig: Type, fdef: FuncDef) -> Type: + if isinstance(sig, FunctionLike): + if fdef.info: + if fdef.info.fullname() == 'mypy_extensions._TypedDict': + # Avoid exposing the internal _TypedDict name. + class_name = 'TypedDict' + else: + class_name = fdef.info.name() + return sig.with_name( + '{} of {}'.format(fdef.name(), class_name)) + else: + return sig.with_name(fdef.name()) + else: + return sig diff --git a/mypy/newsemanal/semanal_typeddict.py b/mypy/newsemanal/semanal_typeddict.py index 0fe5475f5e20..b6a71c7870ce 100644 --- a/mypy/newsemanal/semanal_typeddict.py +++ b/mypy/newsemanal/semanal_typeddict.py @@ -12,7 +12,7 @@ ClassDef, RefExpr, TypeInfo, AssignmentStmt, PassStmt, ExpressionStmt, EllipsisExpr, TempNode, SymbolTableNode, DictExpr, GDEF, ARG_POS, ARG_NAMED ) -from mypy.semanal_shared import SemanticAnalyzerInterface +from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.options import Options from mypy.newsemanal.typeanal import check_for_explicit_any, has_any_from_unimported_type @@ -129,9 +129,12 @@ def check_typeddict_classdef(self, defn: ClassDef, continue # Append name and type in this case... fields.append(name) - types.append(AnyType(TypeOfAny.unannotated) - if stmt.type is None - else self.api.anal_type(stmt.type)) + if stmt.type is None: + types.append(AnyType(TypeOfAny.unannotated)) + else: + analyzed = self.api.anal_type(stmt.type) + assert analyzed is not None # TODO: Handle None values + types.append(analyzed) # ...despite possible minor failures that allow further analyzis. if stmt.type is None or hasattr(stmt, 'new_syntax') and not stmt.new_syntax: self.fail(TPDICT_CLASS_ERROR, stmt) @@ -264,7 +267,9 @@ def parse_typeddict_fields_with_types( except TypeTranslationError: self.fail_typeddict_arg('Invalid field type', field_type_expr) return [], [], False - types.append(self.api.anal_type(type)) + analyzed = self.api.anal_type(type) + assert analyzed is not None # TODO: Handle None values + types.append(analyzed) return items, types, True def fail_typeddict_arg(self, message: str, diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py index fe7ca64d5b4f..92b4b6eab5a6 100644 --- a/mypy/newsemanal/typeanal.py +++ b/mypy/newsemanal/typeanal.py @@ -28,7 +28,7 @@ from mypy.tvar_scope import TypeVarScope from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.plugin import Plugin, TypeAnalyzerPluginInterface, AnalyzeTypeContext -from mypy.semanal_shared import SemanticAnalyzerCoreInterface +from mypy.newsemanal.semanal_shared import SemanticAnalyzerCoreInterface from mypy import nodes, message_registry MYPY = False diff --git a/mypy/options.py b/mypy/options.py index f063cf444c83..5ce766929a78 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -52,7 +52,7 @@ class BuildType: } # type: Final OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS | - {"platform", "bazel", "plugins"}) + {"platform", "bazel", "plugins", "new_semantic_analyzer"}) - {"debug_cache"}) # type: Final @@ -84,6 +84,9 @@ def __init__(self) -> None: # PEP 420 namespace packages self.namespace_packages = False + # Use the new semantic analyzer + self.new_semantic_analyzer = False + # disallow_any options self.disallow_any_generics = False self.disallow_any_unimported = False diff --git a/mypy/plugin.py b/mypy/plugin.py index 40783ddcc5d3..fba43a29d2aa 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -181,7 +181,7 @@ def anal_type(self, t: Type, *, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, report_invalid_types: bool = True, - third_pass: bool = False) -> Type: + third_pass: bool = False) -> Optional[Type]: """Analyze an unbound type.""" raise NotImplementedError diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index eb7729c2106e..dd9d6f25ab84 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -82,6 +82,7 @@ 'check-final.test', 'check-redefine.test', 'check-literal.test', + 'check-newsemanal.test', ] diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test new file mode 100644 index 000000000000..56f5dacbd851 --- /dev/null +++ b/test-data/unit/check-newsemanal.test @@ -0,0 +1,85 @@ +-- Test cases for the new semantic analyzer + +[case testNewAnalyzerEmpty] +# flags: --new-semantic-analyzer + +[case testNewAnalyzerSimpleAssignment] +# flags: --new-semantic-analyzer +x = 1 +x.y # E: "int" has no attribute "y" +y # E: Name 'y' is not defined + +[case testNewAnalyzerSimpleAnnotation] +# flags: --new-semantic-analyzer +x: int = 0 +y: str = 0 \ + # E: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testNewAnalyzerSimpleClass] +# flags: --new-semantic-analyzer +class A: + x: int +a: A +a.x +a.y # E: "A" has no attribute "y" + +[case testNewAnalyzerErrorInClassBody] +# flags: --new-semantic-analyzer +class A: + x # E: Name 'x' is not defined + +[case testNewAnalyzerTypeAnnotationForwardReference] +# flags: --new-semantic-analyzer +class A: + b: B +class B: + a: A +a: A +b: B +a.b = a # E: Incompatible types in assignment (expression has type "A", variable has type "B") +a.b = b +b.a = a +b.a = b # E: Incompatible types in assignment (expression has type "B", variable has type "A") + +[case testNewAnalyzerTypeAnnotationCycle1] +# flags: --new-semantic-analyzer +import a +[file a.py] +import b +class A: pass +y: b.B +y() +[file b.py] +import a +class B: pass +x: a.A +x() +[out] +tmp/b.py:4: error: "A" not callable +tmp/a.py:4: error: "B" not callable + +[case testNewAnalyzerTypeAnnotationCycle2] +# flags: --new-semantic-analyzer +import a +[file a.py] +from b import B +class A: pass +y: B +y() +[file b.py] +from a import A +class B: pass +x: A +x() +[out] +tmp/b.py:4: error: "A" not callable +tmp/a.py:4: error: "B" not callable + +[case testNewAnalyzerTypeAnnotationCycle3] +# flags: --new-semantic-analyzer +import b +[file a.py] +from b import bad # E: Module 'b' has no attribute 'bad' +[file b.py] +from a import bad2 # E: Module 'a' has no attribute 'bad2' +[out]