Unify test suites' module exclusion logic

mypy/build.py

@@ -705,8 +705,8 @@ def __init__(
         self.quickstart_state = read_quickstart_file(options, self.stdout)
         # Fine grained targets (module top levels and top level functions) processed by
         # the semantic analyzer, used only for testing. Currently used only by the new
-        # semantic analyzer.
-        self.processed_targets: list[str] = []
+        # semantic analyzer. Tuple of module and target name.
+        self.processed_targets: list[tuple[str, str]] = []
         # Missing stub packages encountered.
         self.missing_stub_packages: set[str] = set()
         # Cache for mypy ASTs that have completed semantic analysis

mypy/semanal_main.py

@@ -218,7 +218,7 @@ def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None:
             state = graph[next_id]
             assert state.tree is not None
             deferred, incomplete, progress = semantic_analyze_target(
-                next_id, state, state.tree, None, final_iteration, patches
+                next_id, next_id, state, state.tree, None, final_iteration, patches
             )
             all_deferred += deferred
             any_progress = any_progress or progress
@@ -289,7 +289,7 @@ def process_top_level_function(
             # OK, this is one last pass, now missing names will be reported.
             analyzer.incomplete_namespaces.discard(module)
         deferred, incomplete, progress = semantic_analyze_target(
-            target, state, node, active_type, final_iteration, patches
+            target, module, state, node, active_type, final_iteration, patches
         )
         if final_iteration:
             assert not deferred, "Must not defer during final iteration"
@@ -318,6 +318,7 @@ def get_all_leaf_targets(file: MypyFile) -> list[TargetInfo]:
 
 def semantic_analyze_target(
     target: str,
+    module: str,
     state: State,
     node: MypyFile | FuncDef | OverloadedFuncDef | Decorator,
     active_type: TypeInfo | None,
@@ -331,7 +332,7 @@ def semantic_analyze_target(
     - was some definition incomplete (need to run another pass)
     - were any new names defined (or placeholders replaced)
     """
-    state.manager.processed_targets.append(target)
+    state.manager.processed_targets.append((module, target))
     tree = state.tree
     assert tree is not None
     analyzer = state.manager.semantic_analyzer

mypy/test/data.py

@@ -41,6 +41,14 @@ class DeleteFile(NamedTuple):
 FileOperation: _TypeAlias = Union[UpdateFile, DeleteFile]
 
 
+def _file_arg_to_module(filename: str) -> str:
+    filename, _ = os.path.splitext(filename)
+    parts = filename.split("/")  # not os.sep since it comes from test data
+    if parts[-1] == "__init__":
+        parts.pop()
+    return ".".join(parts)
+
+
 def parse_test_case(case: DataDrivenTestCase) -> None:
     """Parse and prepare a single case from suite with test case descriptions.
 
@@ -65,22 +73,26 @@ def parse_test_case(case: DataDrivenTestCase) -> None:
     rechecked_modules: dict[int, set[str]] = {}  # from run number module names
     triggered: list[str] = []  # Active triggers (one line per incremental step)
     targets: dict[int, list[str]] = {}  # Fine-grained targets (per fine-grained update)
+    test_modules: list[str] = []  # Modules which are deemed "test" (vs "fixture")
 
     # Process the parsed items. Each item has a header of form [id args],
     # optionally followed by lines of text.
     item = first_item = test_items[0]
+    test_modules.append("__main__")
     for item in test_items[1:]:
-        if item.id in {"file", "outfile", "outfile-re"}:
+        if item.id in {"file", "fixture", "outfile", "outfile-re"}:
             # Record an extra file needed for the test case.
             assert item.arg is not None
             contents = expand_variables("\n".join(item.data))
-            file_entry = (join(base_path, item.arg), contents)
-            if item.id == "file":
-                files.append(file_entry)
+            path = join(base_path, item.arg)
+            if item.id != "fixture":
+                test_modules.append(_file_arg_to_module(item.arg))
+            if item.id in {"file", "fixture"}:
+                files.append((path, contents))
             elif item.id == "outfile-re":
-                output_files.append((file_entry[0], re.compile(file_entry[1].rstrip(), re.S)))
-            else:
-                output_files.append(file_entry)
+                output_files.append((path, re.compile(contents.rstrip(), re.S)))
+            elif item.id == "outfile":
+                output_files.append((path, contents))
         elif item.id == "builtins":
             # Use an alternative stub file for the builtins module.
             assert item.arg is not None
@@ -207,6 +219,7 @@ def parse_test_case(case: DataDrivenTestCase) -> None:
     case.triggered = triggered or []
     case.normalize_output = normalize_output
     case.expected_fine_grained_targets = targets
+    case.test_modules = test_modules
 
 
 class DataDrivenTestCase(pytest.Item):
@@ -225,6 +238,8 @@ class DataDrivenTestCase(pytest.Item):
 
     # (file path, file content) tuples
     files: list[tuple[str, str]]
+    # Modules which is to be considered "test" rather than "fixture"
+    test_modules: list[str]
     expected_stale_modules: dict[int, set[str]]
     expected_rechecked_modules: dict[int, set[str]]
     expected_fine_grained_targets: dict[int, list[str]]

mypy/test/testcheck.py

@@ -11,7 +11,6 @@
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths
 from mypy.options import TYPE_VAR_TUPLE, UNPACK
-from mypy.semanal_main import core_modules
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path
 from mypy.test.helpers import (
@@ -188,12 +187,10 @@ def run_case_once(
             if incremental_step:
                 name += str(incremental_step + 1)
             expected = testcase.expected_fine_grained_targets.get(incremental_step + 1)
-            actual = res.manager.processed_targets
-            # Skip the initial builtin cycle.
             actual = [
-                t
-                for t in actual
-                if not any(t.startswith(mod) for mod in core_modules + ["mypy_extensions"])
+                target
+                for module, target in res.manager.processed_targets
+                if module in testcase.test_modules
             ]
             if expected is not None:
                 assert_target_equivalence(name, expected, actual)

mypy/test/testdeps.py

@@ -41,16 +41,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                 a = ["Unknown compile error (likely syntax error in test case or fixture)"]
         else:
             deps: defaultdict[str, set[str]] = defaultdict(set)
-            for module in files:
-                if (
-                    module in dumped_modules
-                    or dump_all
-                    and module
-                    not in ("abc", "typing", "mypy_extensions", "typing_extensions", "enum")
-                ):
-                    new_deps = get_dependencies(
-                        files[module], type_map, options.python_version, options
-                    )
+            for module, file in files.items():
+                if (module in dumped_modules or dump_all) and (module in testcase.test_modules):
+                    new_deps = get_dependencies(file, type_map, options.python_version, options)
                     for source in new_deps:
                         deps[source].update(new_deps[source])
 

mypy/test/testmerge.py

@@ -36,18 +36,6 @@
 AST = "AST"
 
 
-NOT_DUMPED_MODULES = (
-    "builtins",
-    "typing",
-    "abc",
-    "contextlib",
-    "sys",
-    "mypy_extensions",
-    "typing_extensions",
-    "enum",
-)
-
-
 class ASTMergeSuite(DataSuite):
     files = ["merge.test"]
 
@@ -84,13 +72,13 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         target_path = os.path.join(test_temp_dir, "target.py")
         shutil.copy(os.path.join(test_temp_dir, "target.py.next"), target_path)
 
-        a.extend(self.dump(fine_grained_manager, kind))
+        a.extend(self.dump(fine_grained_manager, kind, testcase.test_modules))
         old_subexpr = get_subexpressions(result.manager.modules["target"])
 
         a.append("==>")
 
         new_file, new_types = self.build_increment(fine_grained_manager, "target", target_path)
-        a.extend(self.dump(fine_grained_manager, kind))
+        a.extend(self.dump(fine_grained_manager, kind, testcase.test_modules))
 
         for expr in old_subexpr:
             if isinstance(expr, TypeVarExpr):
@@ -137,34 +125,32 @@ def build_increment(
         type_map = manager.graph[module_id].type_map()
         return module, type_map
 
-    def dump(self, manager: FineGrainedBuildManager, kind: str) -> list[str]:
-        modules = manager.manager.modules
+    def dump(
+        self, manager: FineGrainedBuildManager, kind: str, test_modules: list[str]
+    ) -> list[str]:
+        modules = {
+            name: file for name, file in manager.manager.modules.items() if name in test_modules
+        }
         if kind == AST:
             return self.dump_asts(modules)
         elif kind == TYPEINFO:
             return self.dump_typeinfos(modules)
         elif kind == SYMTABLE:
             return self.dump_symbol_tables(modules)
         elif kind == TYPES:
-            return self.dump_types(manager)
+            return self.dump_types(modules, manager)
         assert False, f"Invalid kind {kind}"
 
     def dump_asts(self, modules: dict[str, MypyFile]) -> list[str]:
         a = []
         for m in sorted(modules):
-            if m in NOT_DUMPED_MODULES:
-                # We don't support incremental checking of changes to builtins, etc.
-                continue
             s = modules[m].accept(self.str_conv)
             a.extend(s.splitlines())
         return a
 
     def dump_symbol_tables(self, modules: dict[str, MypyFile]) -> list[str]:
         a = []
         for id in sorted(modules):
-            if not is_dumped_module(id):
-                # We don't support incremental checking of changes to builtins, etc.
-                continue
             a.extend(self.dump_symbol_table(id, modules[id].names))
         return a
 
@@ -197,8 +183,6 @@ def format_symbol_table_node(self, node: SymbolTableNode) -> str:
     def dump_typeinfos(self, modules: dict[str, MypyFile]) -> list[str]:
         a = []
         for id in sorted(modules):
-            if not is_dumped_module(id):
-                continue
             a.extend(self.dump_typeinfos_recursive(modules[id].names))
         return a
 
@@ -217,13 +201,13 @@ def dump_typeinfo(self, info: TypeInfo) -> list[str]:
         s = info.dump(str_conv=self.str_conv, type_str_conv=self.type_str_conv)
         return s.splitlines()
 
-    def dump_types(self, manager: FineGrainedBuildManager) -> list[str]:
+    def dump_types(
+        self, modules: dict[str, MypyFile], manager: FineGrainedBuildManager
+    ) -> list[str]:
         a = []
         # To make the results repeatable, we try to generate unique and
         # deterministic sort keys.
-        for module_id in sorted(manager.manager.modules):
-            if not is_dumped_module(module_id):
-                continue
+        for module_id in sorted(modules):
             all_types = manager.manager.all_types
             # Compute a module type map from the global type map
             tree = manager.graph[module_id].tree
@@ -242,7 +226,3 @@ def dump_types(self, manager: FineGrainedBuildManager) -> list[str]:
 
     def format_type(self, typ: Type) -> str:
         return typ.accept(self.type_str_conv)
-
-
-def is_dumped_module(id: str) -> bool:
-    return id not in NOT_DUMPED_MODULES and (not id.startswith("_") or id == "__main__")

mypy/test/testsemanal.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import os.path
 import sys
 from typing import Dict
 
@@ -77,27 +76,9 @@ def test_semanal(testcase: DataDrivenTestCase) -> None:
             raise CompileError(a)
         # Include string representations of the source files in the actual
         # output.
-        for fnam in sorted(result.files.keys()):
-            f = result.files[fnam]
-            # Omit the builtins module and files with a special marker in the
-            # path.
-            # TODO the test is not reliable
-            if (
-                not f.path.endswith(
-                    (
-                        os.sep + "builtins.pyi",
-                        "typing.pyi",
-                        "mypy_extensions.pyi",
-                        "typing_extensions.pyi",
-                        "abc.pyi",
-                        "collections.pyi",
-                        "sys.pyi",
-                    )
-                )
-                and not os.path.basename(f.path).startswith("_")
-                and not os.path.splitext(os.path.basename(f.path))[0].endswith("_")
-            ):
-                a += str(f).split("\n")
+        for module in sorted(result.files.keys()):
+            if module in testcase.test_modules:
+                a += str(result.files[module]).split("\n")
     except CompileError as e:
         a = e.messages
     if testcase.normalize_output:
@@ -164,10 +145,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             a = result.errors
             if a:
                 raise CompileError(a)
-            for f in sorted(result.files.keys()):
-                if f not in ("builtins", "typing", "abc"):
-                    a.append(f"{f}:")
-                    for s in str(result.files[f].names).split("\n"):
+            for module in sorted(result.files.keys()):
+                if module in testcase.test_modules:
+                    a.append(f"{module}:")
+                    for s in str(result.files[module].names).split("\n"):
                         a.append("  " + s)
         except CompileError as e:
             a = e.messages
@@ -199,11 +180,13 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
 
             # Collect all TypeInfos in top-level modules.
             typeinfos = TypeInfoMap()
-            for f in result.files.values():
-                for n in f.names.values():
-                    if isinstance(n.node, TypeInfo):
-                        assert n.fullname
-                        typeinfos[n.fullname] = n.node
+            for module, file in result.files.items():
+                if module in testcase.test_modules:
+                    for n in file.names.values():
+                        if isinstance(n.node, TypeInfo):
+                            assert n.fullname
+                            if any(n.fullname.startswith(m + ".") for m in testcase.test_modules):
+                                typeinfos[n.fullname] = n.node
 
             # The output is the symbol table converted into a string.
             a = str(typeinfos).split("\n")
@@ -220,12 +203,7 @@ class TypeInfoMap(Dict[str, TypeInfo]):
     def __str__(self) -> str:
         a: list[str] = ["TypeInfoMap("]
         for x, y in sorted(self.items()):
-            if (
-                not x.startswith("builtins.")
-                and not x.startswith("typing.")
-                and not x.startswith("abc.")
-            ):
-                ti = ("\n" + "  ").join(str(y).split("\n"))
-                a.append(f"  {x} : {ti}")
+            ti = ("\n" + "  ").join(str(y).split("\n"))
+            a.append(f"  {x} : {ti}")
         a[-1] += ")"
         return "\n".join(a)

mypy/test/testtransform.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import os.path
-
 from mypy import build
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource
@@ -50,29 +48,12 @@ def test_transform(testcase: DataDrivenTestCase) -> None:
             raise CompileError(a)
         # Include string representations of the source files in the actual
         # output.
-        for fnam in sorted(result.files.keys()):
-            f = result.files[fnam]
-
-            # Omit the builtins module and files with a special marker in the
-            # path.
-            # TODO the test is not reliable
-            if (
-                not f.path.endswith(
-                    (
-                        os.sep + "builtins.pyi",
-                        "typing_extensions.pyi",
-                        "typing.pyi",
-                        "abc.pyi",
-                        "sys.pyi",
-                    )
-                )
-                and not os.path.basename(f.path).startswith("_")
-                and not os.path.splitext(os.path.basename(f.path))[0].endswith("_")
-            ):
+        for module in sorted(result.files.keys()):
+            if module in testcase.test_modules:
                 t = TypeAssertTransformVisitor()
                 t.test_only = True
-                f = t.mypyfile(f)
-                a += str(f).split("\n")
+                file = t.mypyfile(result.files[module])
+                a += str(file).split("\n")
     except CompileError as e:
         a = e.messages
     if testcase.normalize_output: