Hide imported names in stubs unless 'as id' is used (PEP 484) (#3706)

Fixes #2927

PEP 484 specifies that only names imported as
``from mod import name as name`` should be re-exported.
However, mypy didn't follow this rule so that this code
passed without errors but obviously fails at runtime:

from collections import TypeVar
from weakref import Generic
from getopt import List

T = TypeVar('T')

class C(Generic[T], List[int]):
    ...

This PR makes all these errors.

Author: ilevkivskyi

mypy/nodes.py

@@ -2286,6 +2286,8 @@ class SymbolTableNode:
     # If False, this name won't be imported via 'from <module> import *'.
     # This has no effect on names within classes.
     module_public = True
+    # If True, the name will be never exported (needed for stub files)
+    module_hidden = False
     # For deserialized MODULE_REF nodes, the referenced module name;
     # for other nodes, optionally the name of the referenced object.
     cross_ref = None  # type: Optional[str]
@@ -2302,11 +2304,13 @@ def __init__(self,
                  module_public: bool = True,
                  normalized: bool = False,
                  alias_tvars: Optional[List[str]] = None,
-                 implicit: bool = False) -> None:
+                 implicit: bool = False,
+                 module_hidden: bool = False) -> None:
         self.kind = kind
         self.node = node
         self.type_override = typ
         self.mod_id = mod_id
+        self.module_hidden = module_hidden
         self.module_public = module_public
         self.normalized = normalized
         self.alias_tvars = alias_tvars
@@ -2352,6 +2356,8 @@ def serialize(self, prefix: str, name: str) -> JsonDict:
         data = {'.class': 'SymbolTableNode',
                 'kind': node_kinds[self.kind],
                 }  # type: JsonDict
+        if self.module_hidden:
+            data['module_hidden'] = True
         if not self.module_public:
             data['module_public'] = False
         if self.normalized:
@@ -2393,6 +2399,8 @@ def deserialize(cls, data: JsonDict) -> 'SymbolTableNode':
             stnode = SymbolTableNode(kind, node, typ=typ)
             if 'alias_tvars' in data:
                 stnode.alias_tvars = data['alias_tvars']
+        if 'module_hidden' in data:
+            stnode.module_hidden = data['module_hidden']
         if 'module_public' in data:
             stnode.module_public = data['module_public']
         if 'normalized' in data:

mypy/semanal.py

@@ -1317,12 +1317,11 @@ def visit_import(self, i: Import) -> None:
             if as_id is not None:
                 self.add_module_symbol(id, as_id, module_public=True, context=i)
             else:
-                # Modules imported in a stub file without using 'as x' won't get exported when
-                # doing 'from m import *'.
+                # Modules imported in a stub file without using 'as x' won't get exported
                 module_public = not self.is_stub_file
                 base = id.split('.')[0]
                 self.add_module_symbol(base, base, module_public=module_public,
-                                       context=i)
+                                       context=i, module_hidden=not module_public)
                 self.add_submodules_to_parent_modules(id, module_public)
 
     def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None:
@@ -1351,11 +1350,12 @@ def add_submodules_to_parent_modules(self, id: str, module_public: bool) -> None
             id = parent
 
     def add_module_symbol(self, id: str, as_id: str, module_public: bool,
-                          context: Context) -> None:
+                          context: Context, module_hidden: bool = False) -> None:
         if id in self.modules:
             m = self.modules[id]
             self.add_symbol(as_id, SymbolTableNode(MODULE_REF, m, self.cur_mod_id,
-                                                   module_public=module_public), context)
+                                                   module_public=module_public,
+                                                   module_hidden=module_hidden), context)
         else:
             self.add_unknown_symbol(as_id, context, is_import=True)
 
@@ -1366,11 +1366,11 @@ def visit_import_from(self, imp: ImportFrom) -> None:
         for id, as_id in imp.names:
             node = module.names.get(id) if module else None
             missing = False
+            possible_module_id = import_id + '.' + id
 
             # If the module does not contain a symbol with the name 'id',
             # try checking if it's a module instead.
             if not node or node.kind == UNBOUND_IMPORTED:
-                possible_module_id = import_id + '.' + id
                 mod = self.modules.get(possible_module_id)
                 if mod is not None:
                     node = SymbolTableNode(MODULE_REF, mod, import_id)
@@ -1394,7 +1394,7 @@ def visit_import_from(self, imp: ImportFrom) -> None:
                     symbol = SymbolTableNode(GDEF, ast_node, name)
                     self.add_symbol(name, symbol, imp)
                     return
-            if node and node.kind != UNBOUND_IMPORTED:
+            if node and node.kind != UNBOUND_IMPORTED and not node.module_hidden:
                 node = self.normalize_type_alias(node, imp)
                 if not node:
                     return
@@ -1407,12 +1407,14 @@ def visit_import_from(self, imp: ImportFrom) -> None:
                         continue
                 # 'from m import x as x' exports x in a stub file.
                 module_public = not self.is_stub_file or as_id is not None
+                module_hidden = not module_public and possible_module_id not in self.modules
                 symbol = SymbolTableNode(node.kind, node.node,
                                          self.cur_mod_id,
                                          node.type_override,
                                          module_public=module_public,
                                          normalized=node.normalized,
-                                         alias_tvars=node.alias_tvars)
+                                         alias_tvars=node.alias_tvars,
+                                         module_hidden=module_hidden)
                 self.add_symbol(imported_id, symbol, imp)
             elif module and not missing:
                 # Missing attribute.
@@ -3164,7 +3166,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None:
             # bar in its namespace.  This must be done for all types of bar.
             file = cast(Optional[MypyFile], base.node)  # can't use isinstance due to issue #2999
             n = file.names.get(expr.name, None) if file is not None else None
-            if n:
+            if n and not n.module_hidden:
                 n = self.normalize_type_alias(n, expr)
                 if not n:
                     return
@@ -3508,7 +3510,11 @@ def lookup_qualified(self, name: str, ctx: Context,
                         break
                 if n:
                     n = self.normalize_type_alias(n, ctx)
-            return n
+                    if n and n.module_hidden:
+                        self.name_not_defined(name, ctx)
+            if n and not n.module_hidden:
+                return n
+            return None
 
     def builtin_type(self, fully_qualified_name: str) -> Instance:
         sym = self.lookup_fully_qualified(fully_qualified_name)

test-data/unit/check-modules.test

@@ -980,7 +980,7 @@ x + ''  # No error here
 y + ''  # No error here
 z + ''  # Error here
 [file stub.pyi]
-from non_stub import x  # this import is not followed
+from non_stub import x as x  # this import is not followed
 
 z = 42
 [file non_stub.py]
@@ -1665,6 +1665,128 @@ m = n  # E: Cannot assign multiple modules to name 'm' without explicit 'types.M
 
 [builtins fixtures/module.pyi]
 
+[case testNoReExportFromStubs]
+from stub import Iterable  # E: Module 'stub' has no attribute 'Iterable'
+from stub import C
+
+c = C()
+reveal_type(c.x)  # E: Revealed type is 'builtins.int'
+it: Iterable[int]
+reveal_type(it)  # E: Revealed type is 'Any'
+
+[file stub.pyi]
+from typing import Iterable
+from substub import C as C
+
+def fun(x: Iterable[str]) -> Iterable[int]: pass
+
+[file substub.pyi]
+class C:
+    x: int
+
+[builtins fixtures/module.pyi]
+
+[case testNoReExportFromStubsMemberType]
+import stub
+
+c = stub.C()
+reveal_type(c.x)  # E: Revealed type is 'builtins.int'
+it: stub.Iterable[int]  # E: Name 'stub.Iterable' is not defined
+reveal_type(it)  # E: Revealed type is 'Any'
+
+[file stub.pyi]
+from typing import Iterable
+from substub import C as C
+
+def fun(x: Iterable[str]) -> Iterable[int]: pass
+
+[file substub.pyi]
+class C:
+    x: int
+
+[builtins fixtures/module.pyi]
+
+[case testNoReExportFromStubsMemberVar]
+import stub
+
+reveal_type(stub.y)  # E: Revealed type is 'builtins.int'
+reveal_type(stub.z)  # E: Revealed type is 'Any' \
+                     # E: Module has no attribute "z"
+
+[file stub.pyi]
+from substub import y as y
+from substub import z
+
+[file substub.pyi]
+y = 42
+z: int
+
+[builtins fixtures/module.pyi]
+
+[case testReExportChildStubs]
+import mod
+from mod import submod
+
+reveal_type(mod.x)  # E: Revealed type is 'mod.submod.C'
+y = submod.C()
+reveal_type(y.a)  # E: Revealed type is 'builtins.str'
+
+[file mod/__init__.pyi]
+from . import submod
+x: submod.C
+
+[file mod/submod.pyi]
+class C:
+    a: str
+
+[builtins fixtures/module.pyi]
+
+[case testReExportChildStubs2]
+import mod.submod
+
+y = mod.submod.C()
+reveal_type(y.a)  # E: Revealed type is 'builtins.str'
+
+[file mod/__init__.pyi]
+from . import submod
+x: submod.C
+
+[file mod/submod.pyi]
+class C:
+    a: str
+
+[builtins fixtures/module.pyi]
+
+[case testNoReExportChildStubs]
+import mod
+from mod import C, D  # E: Module 'mod' has no attribute 'C'
+
+reveal_type(mod.x)  # E: Revealed type is 'mod.submod.C'
+mod.C  # E: Module has no attribute "C"
+y = mod.D()
+reveal_type(y.a)  # E: Revealed type is 'builtins.str'
+
+[file mod/__init__.pyi]
+from .submod import C, D as D
+x: C
+
+[file mod/submod.pyi]
+class C: pass
+class D:
+    a: str
+[builtins fixtures/module.pyi]
+
+[case testNoReExportNestedStub]
+from stub import substub  # E: Module 'stub' has no attribute 'substub'
+
+[file stub.pyi]
+import substub
+
+[file substub.pyi]
+x = 42
+
+[file mod/submod.pyi]
+
 [case testModuleAliasToQualifiedImport]
 import package.module
 alias = package.module