Caching for fine-grained incremental mode

mypy/test/testfinegrained.py

@@ -30,6 +30,7 @@
 from mypy.test.testtypegen import ignore_node
 from mypy.types import TypeStrVisitor, Type
 from mypy.util import short_type
+import pytest  # type: ignore  # no pytest in typeshed
 
 
 class FineGrainedSuite(DataSuite):
@@ -41,31 +42,40 @@ class FineGrainedSuite(DataSuite):
     ]
     base_path = test_temp_dir
     optional_out = True
+    # Whether to use the fine-grained cache in the testing. This is overridden
+    # by a trivial subclass to produce a suite that uses the cache.
+    use_cache = False
+
+    # Decide whether to skip the test. This could have been structured
+    # as a filter() classmethod also, but we want the tests reported
+    # as skipped, not just elided.
+    def should_skip(self, testcase: DataDrivenTestCase) -> bool:
+        if self.use_cache:
+            if testcase.name.endswith("-skip-cache"): return True
+            # TODO: In caching mode we currently don't well support
+            # starting from cached states with errors in them.
+            if testcase.output and testcase.output[0] != '==': return True
+        else:
+            if testcase.name.endswith("-skip-nocache"): return True
 
-    def run_case(self, testcase: DataDrivenTestCase) -> None:
-        self.run_case_inner(testcase, cache=False)
-
-        # Reset the test case and run it again with caching on
-        testcase.teardown()
-        testcase.setup()
-        self.run_case_inner(testcase, cache=True)
+        return False
 
-    def run_case_inner(self, testcase: DataDrivenTestCase, cache: bool) -> None:
-        # TODO: In caching mode we currently don't well support
-        # starting from cached states with errors in them.
-        if cache and testcase.output and testcase.output[0] != '==': return
+    def run_case(self, testcase: DataDrivenTestCase) -> None:
+        if self.should_skip(testcase):
+            pytest.skip()
+            return
 
         main_src = '\n'.join(testcase.input)
         sources_override = self.parse_sources(main_src)
-        print("Testing with cache: ", cache)
         messages, manager, graph = self.build(main_src, testcase, sources_override,
-                                              build_cache=cache, enable_cache=cache)
+                                              build_cache=self.use_cache,
+                                              enable_cache=self.use_cache)
         a = []
         if messages:
             a.extend(normalize_messages(messages))
 
         fine_grained_manager = None
-        if not cache:
+        if not self.use_cache:
             fine_grained_manager = FineGrainedBuildManager(manager, graph)
 
         steps = testcase.find_steps()
@@ -90,7 +100,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase, cache: bool) -> None:
             # cache, now we need to set it up
             if fine_grained_manager is None:
                 messages, manager, graph = self.build(main_src, testcase, sources_override,
-                                                      build_cache=False, enable_cache=cache)
+                                                      build_cache=False, enable_cache=True)
                 fine_grained_manager = FineGrainedBuildManager(manager, graph)
 
             new_messages = fine_grained_manager.update(modules)
@@ -103,11 +113,10 @@ def run_case_inner(self, testcase: DataDrivenTestCase, cache: bool) -> None:
         # Normalize paths in test output (for Windows).
         a = [line.replace('\\', '/') for line in a]
 
-        modestr = "in cache mode " if cache else ""
         assert_string_arrays_equal(
             testcase.output, a,
-            'Invalid output {}({}, line {})'.format(
-                modestr, testcase.file, testcase.line))
+            'Invalid output ({}, line {})'.format(
+                testcase.file, testcase.line))
 
         if testcase.triggered:
             assert_string_arrays_equal(

mypy/test/testfinegrainedcache.py

@@ -0,0 +1,12 @@
+"""Tests for fine-grained incremental checking using the cache.
+
+All of the real code for this lives in testfinegrained.py.
+"""
+
+# We can't "import FineGrainedSuite from ..." because that will cause pytest
+# to collect the non-caching tests when running this file.
+import mypy.test.testfinegrained
+
+
+class FineGrainedCacheSuite(mypy.test.testfinegrained.FineGrainedSuite):
+    use_cache = True

runtests.py

@@ -209,6 +209,7 @@ def test_path(*names: str):
     'testdeps',
     'testdiff',
     'testfinegrained',
+    'testfinegrainedcache',
     'testmerge',
     'testtransform',
     'testtypegen',

test-data/unit/fine-grained-blockers.test

@@ -237,8 +237,8 @@ a.py:1: error: invalid syntax
 b.py:3: error: Too many arguments for "f"
 a.py:3: error: Too many arguments for "g"
 
-[case testDeleteFileWithBlockingError-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testDeleteFileWithBlockingError-skip-cache]
+-- Different cache/no-cache tests because:
 -- Error message ordering differs
 import a
 import b
@@ -258,6 +258,27 @@ main:1: error: Cannot find module named 'a'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 b.py:1: error: Cannot find module named 'a'
 
+[case testDeleteFileWithBlockingError-skip-nocache]
+-- Different cache/no-cache tests because:
+-- Error message ordering differs
+import a
+import b
+[file a.py]
+def f() -> None: pass
+[file b.py]
+import a
+a.f()
+[file a.py.2]
+x x
+[delete a.py.3]
+[out]
+==
+a.py:1: error: invalid syntax
+==
+b.py:1: error: Cannot find module named 'a'
+b.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+main:1: error: Cannot find module named 'a'
+
 [case testModifyFileWhileBlockingErrorElsewhere]
 import a
 import b
@@ -310,8 +331,8 @@ import blocker2
 ==
 a.py:1: error: "int" not callable
 
-[case testFixingBlockingErrorTriggersDeletion1-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testFixingBlockingErrorTriggersDeletion1-skip-cache]
+-- Disabled in cache mdode:
 -- Cache mode fails to produce the error in the final step, but this is
 -- a manifestation of a bug that can occur in no-cache mode also.
 import a

test-data/unit/fine-grained-cycles.test

@@ -174,8 +174,8 @@ def h() -> None:
 ==
 a.py:1: error: Module 'b' has no attribute 'C'
 
-[case testReferenceToTypeThroughCycleAndReplaceWithFunction-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testReferenceToTypeThroughCycleAndReplaceWithFunction-skip-cache]
+-- Different cache/no-cache tests because:
 -- Cache mode has a "need type annotation" message (like coarse incremental does)
 
 import a
@@ -208,6 +208,41 @@ def h() -> None:
 ==
 a.py:3: error: Invalid type "b.C"
 
+[case testReferenceToTypeThroughCycleAndReplaceWithFunction-skip-nocache]
+-- Different cache/no-cache tests because:
+-- Cache mode has a "need type annotation" message (like coarse incremental does)
+
+import a
+
+[file a.py]
+from b import C
+
+def f() -> C: pass
+
+[file b.py]
+import a
+
+class C:
+    def g(self) -> None: pass
+
+def h() -> None:
+    c = a.f()
+    c.g()
+
+[file b.py.2]
+import a
+
+def C() -> int: pass
+
+def h() -> None:
+    c = a.f()
+    c.g()
+
+[out]
+==
+a.py:3: error: Invalid type "b.C"
+b.py:6: error: Need type annotation for 'c'
+
 -- TODO: More import cycle:
 --
 -- * "from x import y" through cycle

test-data/unit/fine-grained-modules.test

@@ -234,8 +234,8 @@ main:1: error: Cannot find module named 'a'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 ==
 
-[case testDeletionOfSubmoduleTriggersImportFrom1-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testDeletionOfSubmoduleTriggersImportFrom1-skip-cache]
+-- Different cache/no-cache tests because:
 -- Cache mode matches the message from regular mode and no-cache mode
 -- matches the message from coarse incremental mode...
 from p import q
@@ -250,6 +250,21 @@ main:1: error: Cannot find module named 'p.q'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 ==
 
+[case testDeletionOfSubmoduleTriggersImportFrom1-skip-nocache]
+-- Different cache/no-cache tests because:
+-- Cache mode matches the message from regular mode and no-cache mode
+-- matches the message from coarse incremental mode...
+from p import q
+[file p/__init__.py]
+[file p/q.py]
+[delete p/q.py.2]
+[file p/q.py.3]
+[out]
+==
+main:1: error: Module 'p' has no attribute 'q'
+==
+
+
 [case testDeletionOfSubmoduleTriggersImportFrom2]
 from p.q import f
 f()
@@ -733,8 +748,8 @@ a/b.py:3: error: Revealed type is 'Any'
 ==
 a/b.py:3: error: Unsupported operand types for + ("int" and "str")
 
-[case testDeleteModuleWithinPackageInitIgnored-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testDeleteModuleWithinPackageInitIgnored-skip-cache]
+-- Disabled in cache mode because incorrect behavior:
 -- Having deleted files specified on command line seems dodgy, though.
 # cmd: mypy x.py a/b.py
 # flags: --follow-imports=skip --ignore-missing-imports

test-data/unit/fine-grained.test

@@ -286,8 +286,8 @@ main:5: error: Module has no attribute "B"
 ==
 main:5: error: Module has no attribute "B"
 
-[case testContinueToReportErrorAtTopLevel-skip]
--- Disabled due to cache/no-cache mismatch:
+[case testContinueToReportErrorAtTopLevel-skip-cache]
+-- Different cache/no-cache tests because:
 -- Error message ordering differs
 import n
 import m
@@ -311,6 +311,31 @@ n.py:2: error: "A" has no attribute "g"
 ==
 n.py:2: error: "A" has no attribute "g"
 
+[case testContinueToReportErrorAtTopLevel-skip-nocache]
+-- Different cache/no-cache tests because:
+-- Error message ordering differs
+import n
+import m
+m.A().f()
+[file n.py]
+import m
+m.A().g()
+[file m.py]
+class A:
+    def f(self) -> None: pass
+    def g(self) -> None: pass
+[file m.py.2]
+class A: pass
+[file m.py.3]
+class A:
+    def f(self) -> None: pass
+[out]
+==
+n.py:2: error: "A" has no attribute "g"
+main:3: error: "A" has no attribute "f"
+==
+n.py:2: error: "A" has no attribute "g"
+
 [case testContinueToReportErrorInMethod]
 import m
 class C: