Fix caching of PEP 561 namespace packages (#13124)

Fixes #13085. Hopefully more robust than previous fixes along these lines.

Co-authored-by: hauntsaninja <>

Author: hauntsaninja

mypy/build.py

@@ -708,19 +708,12 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
             return new_id
 
         res: List[Tuple[int, str, int]] = []
-        delayed_res: List[Tuple[int, str, int]] = []
         for imp in file.imports:
             if not imp.is_unreachable:
                 if isinstance(imp, Import):
                     pri = import_priority(imp, PRI_MED)
                     ancestor_pri = import_priority(imp, PRI_LOW)
                     for id, _ in imp.ids:
-                        # We append the target (e.g. foo.bar.baz) before the ancestors (e.g. foo
-                        # and foo.bar) so that, if FindModuleCache finds the target module in a
-                        # package marked with py.typed underneath a namespace package installed in
-                        # site-packages, (gasp), that cache's knowledge of the ancestors
-                        # (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find
-                        # the parent.
                         res.append((pri, id, imp.line))
                         ancestor_parts = id.split(".")[:-1]
                         ancestors = []
@@ -729,15 +722,13 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
                             res.append((ancestor_pri, ".".join(ancestors), imp.line))
                 elif isinstance(imp, ImportFrom):
                     cur_id = correct_rel_imp(imp)
-                    any_are_submodules = False
                     all_are_submodules = True
                     # Also add any imported names that are submodules.
                     pri = import_priority(imp, PRI_MED)
                     for name, __ in imp.names:
                         sub_id = cur_id + '.' + name
                         if self.is_module(sub_id):
                             res.append((pri, sub_id, imp.line))
-                            any_are_submodules = True
                         else:
                             all_are_submodules = False
                     # Add cur_id as a dependency, even if all of the
@@ -747,19 +738,18 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
                     # if all of the imports are submodules, do the import at a lower
                     # priority.
                     pri = import_priority(imp, PRI_HIGH if not all_are_submodules else PRI_LOW)
-                    # The imported module goes in after the submodules, for the same namespace
-                    # related reasons discussed in the Import case.
-                    # There is an additional twist: if none of the submodules exist,
-                    # we delay the import in case other imports of other submodules succeed.
-                    if any_are_submodules:
-                        res.append((pri, cur_id, imp.line))
-                    else:
-                        delayed_res.append((pri, cur_id, imp.line))
+                    res.append((pri, cur_id, imp.line))
                 elif isinstance(imp, ImportAll):
                     pri = import_priority(imp, PRI_HIGH)
                     res.append((pri, correct_rel_imp(imp), imp.line))
 
-        res.extend(delayed_res)
+        # Sort such that module (e.g. foo.bar.baz) comes before its ancestors (e.g. foo
+        # and foo.bar) so that, if FindModuleCache finds the target module in a
+        # package marked with py.typed underneath a namespace package installed in
+        # site-packages, (gasp), that cache's knowledge of the ancestors
+        # (aka FindModuleCache.ns_ancestors) can be primed when it is asked to find
+        # the parent.
+        res.sort(key=lambda x: -x[1].count("."))
         return res
 
     def is_module(self, id: str) -> bool:

test-data/unit/check-errorcodes.test

@@ -598,11 +598,11 @@ if int() is str():  # E: Non-overlapping identity check (left operand type: "int
 [case testErrorCodeMissingModule]
 from defusedxml import xyz  # E: Cannot find implementation or library stub for module named "defusedxml"  [import]
 from nonexistent import foobar  # E: Cannot find implementation or library stub for module named "nonexistent"  [import]
-import nonexistent2  # E: Cannot find implementation or library stub for module named "nonexistent2"  [import] \
-                     # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+import nonexistent2  # E: Cannot find implementation or library stub for module named "nonexistent2"  [import]
 from nonexistent3 import *  # E: Cannot find implementation or library stub for module named "nonexistent3"  [import]
 from pkg import bad  # E: Module "pkg" has no attribute "bad"  [attr-defined]
-from pkg.bad2 import bad3  # E: Cannot find implementation or library stub for module named "pkg.bad2"  [import]
+from pkg.bad2 import bad3  # E: Cannot find implementation or library stub for module named "pkg.bad2"  [import] \
+                           # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 [file pkg/__init__.py]
 
 [case testErrorCodeAlreadyDefined]

test-data/unit/check-modules.test

@@ -913,10 +913,10 @@ a.b.c.value
 [file a/b/c.py]
 value = 3.2
 [out]
-tmp/a/b/__init__.py:2: error: Name "c" is not defined
-tmp/a/b/__init__.py:3: error: Name "a" is not defined
 tmp/a/__init__.py:2: error: Name "b" is not defined
 tmp/a/__init__.py:3: error: Name "a" is not defined
+tmp/a/b/__init__.py:2: error: Name "c" is not defined
+tmp/a/b/__init__.py:3: error: Name "a" is not defined
 
 [case testSubmoduleMixingLocalAndQualifiedNames]
 from a.b import MyClass

test-data/unit/cmdline.test

@@ -431,19 +431,19 @@ main.py:4: note: Revealed type is "Any"
 [case testConfigFollowImportsError]
 # cmd: mypy main.py
 [file main.py]
-from a import x
+from a import x  # Error reported here
 reveal_type(x)  # Expect Any
-import a  # Error reported here
+import a
 reveal_type(a.x)  # Expect Any
 [file mypy.ini]
 \[mypy]
 follow_imports = error
 [file a.py]
 /  # No error reported
 [out]
+main.py:1: error: Import of "a" ignored
+main.py:1: note: (Using --follow-imports=error, module not passed on command line)
 main.py:2: note: Revealed type is "Any"
-main.py:3: error: Import of "a" ignored
-main.py:3: note: (Using --follow-imports=error, module not passed on command line)
 main.py:4: note: Revealed type is "Any"
 
 [case testConfigFollowImportsSelective]

test-data/unit/fine-grained-follow-imports.test

@@ -530,8 +530,8 @@ def f() -> None: pass
 [out]
 main.py:1: error: Cannot find implementation or library stub for module named "p1.s1.m"
 main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-main.py:1: error: Cannot find implementation or library stub for module named "p1"
 main.py:1: error: Cannot find implementation or library stub for module named "p1.s1"
+main.py:1: error: Cannot find implementation or library stub for module named "p1"
 main.py:2: error: Cannot find implementation or library stub for module named "p2.s2"
 ==
 main.py:2: error: Cannot find implementation or library stub for module named "p2.s2"

test-data/unit/pep561.test

@@ -207,9 +207,19 @@ testNamespacePkgWStubsWithNamespacePackagesFlag.py:8: error: Argument 1 to "bf"
 
 [case testTypedPkgNamespaceRegFromImportTwiceMissing]
 # pkgs: typedpkg_ns_a
-from typedpkg_ns import b  # type: ignore
+from typedpkg_ns import does_not_exist  # type: ignore
 from typedpkg_ns import a
 -- dummy should trigger a second iteration
 [file dummy.py.2]
 [out]
 [out2]
+
+
+[case testTypedPkgNamespaceRegFromImportTwiceMissing2]
+# pkgs: typedpkg_ns_a
+from typedpkg_ns import does_not_exist  # type: ignore
+from typedpkg_ns.a.bbb import bf
+-- dummy should trigger a second iteration
+[file dummy.py.2]
+[out]
+[out2]

test-data/unit/semanal-errors.test

@@ -293,8 +293,8 @@ from m.n import x
 from a.b import *
 [out]
 main:2: error: Cannot find implementation or library stub for module named "m.n"
+main:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 main:3: error: Cannot find implementation or library stub for module named "a.b"
-main:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 
 [case testErrorInImportedModule]
 import m