Allow multiple shadow files to be specified (#5023)

Author: garetht

docs/source/command_line.rst

@@ -463,12 +463,20 @@ Here are some more useful flags:
 
 .. _shadow-file:
 
-- ``--shadow-file SOURCE_FILE SHADOW_FILE`` makes mypy typecheck SHADOW_FILE in
-  place of SOURCE_FILE.  Primarily intended for tooling.  Allows tooling to
-  make transformations to a file before type checking without having to change
-  the file in-place.  (For example, tooling could use this to display the type
-  of an expression by wrapping it with a call to reveal_type in the shadow
-  file and then parsing the output.)
+- ``--shadow-file SOURCE_FILE SHADOW_FILE``: when mypy is asked to typecheck
+  ``SOURCE_FILE``, this makes it read from and typecheck the contents of
+  ``SHADOW_FILE`` instead. However, diagnostics will continue to refer to
+  ``SOURCE_FILE``. Specifying this argument multiple times
+  (``--shadow-file X1 Y1 --shadow-file X2 Y2``)
+  will allow mypy to perform multiple substitutions.
+
+  This allows tooling to create temporary files with helpful modifications
+  without having to change the source file in place. For example, suppose we
+  have a pipeline that adds ``reveal_type`` for certain variables.
+  This pipeline is run on ``original.py`` to produce ``temp.py``.
+  Running ``mypy --shadow-file original.py temp.py original.py`` will then
+  cause mypy to typecheck the contents of ``temp.py`` instead of  ``original.py``,
+  but error messages will still reference ``original.py``.
 
 .. _no-implicit-optional:
 

mypy/build.py

@@ -655,14 +655,34 @@ def __init__(self, data_dir: str,
         self.fscache = fscache
         self.find_module_cache = FindModuleCache(self.fscache)
 
+        # a mapping from source files to their corresponding shadow files
+        # for efficient lookup
+        self.shadow_map = {}  # type: Dict[str, str]
+        if self.options.shadow_file is not None:
+            self.shadow_map = {source_file: shadow_file
+                               for (source_file, shadow_file)
+                               in self.options.shadow_file}
+        # a mapping from each file being typechecked to its possible shadow file
+        self.shadow_equivalence_map = {}  # type: Dict[str, Optional[str]]
+
     def use_fine_grained_cache(self) -> bool:
         return self.cache_enabled and self.options.use_fine_grained_cache
 
     def maybe_swap_for_shadow_path(self, path: str) -> str:
-        if (self.options.shadow_file and
-                os.path.samefile(self.options.shadow_file[0], path)):
-            path = self.options.shadow_file[1]
-        return path
+        if not self.shadow_map:
+            return path
+
+        previously_checked = path in self.shadow_equivalence_map
+        if not previously_checked:
+            for source, shadow in self.shadow_map.items():
+                if self.fscache.samefile(path, source):
+                    self.shadow_equivalence_map[path] = shadow
+                    break
+                else:
+                    self.shadow_equivalence_map[path] = None
+
+        shadow_file = self.shadow_equivalence_map.get(path)
+        return shadow_file if shadow_file else path
 
     def get_stat(self, path: str) -> os.stat_result:
         return self.fscache.stat(self.maybe_swap_for_shadow_path(path))

mypy/fscache.py

@@ -149,6 +149,11 @@ def md5(self, path: str) -> str:
             self.read(path)
         return self.hash_cache[path]
 
+    def samefile(self, f1: str, f2: str) -> bool:
+        s1 = self.stat(f1)
+        s2 = self.stat(f2)
+        return os.path.samestat(s1, s2)  # type: ignore
+
 
 def copy_os_error(e: OSError) -> OSError:
     new = OSError(*e.args)

mypy/main.py

@@ -463,8 +463,9 @@ def add_invertible_flag(flag: str,
     parser.add_argument('--strict', action='store_true', dest='special-opts:strict',
                         help=strict_help)
     parser.add_argument('--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
-                        dest='shadow_file',
-                        help='Typecheck SHADOW_FILE in place of SOURCE_FILE.')
+                        dest='shadow_file', action='append',
+                        help="When encountering SOURCE_FILE, read and typecheck "
+                             "the contents of SHADOW_FILE instead.")
     # hidden options
     # --debug-cache will disable any cache-related compressions/optimizations,
     # which will make the cache writing process output pretty-printed JSON (which

mypy/options.py

@@ -185,7 +185,7 @@ def __init__(self) -> None:
         self.use_builtins_fixtures = False
 
         # -- experimental options --
-        self.shadow_file = None  # type: Optional[Tuple[str, str]]
+        self.shadow_file = None  # type: Optional[List[Tuple[str, str]]]
         self.show_column_numbers = False  # type: bool
         self.dump_graph = False
         self.dump_deps = False

test-data/unit/cmdline.test

@@ -1126,6 +1126,45 @@ follow_imports_for_stubs = True
 import math
 math.frobnicate()
 
+[case testShadowFile1]
+# cmd: mypy --shadow-file source.py shadow.py source.py
+[file source.py]
+def foo() -> str:
+    return "bar"
+[file shadow.py]
+def bar() -> str:
+    return 14
+[out]
+source.py:2: error: Incompatible return value type (got "int", expected "str")
+
+[case testShadowFile2]
+# cmd: mypy --shadow-file s1.py shad1.py --shadow-file s2.py shad2.py --shadow-file s3.py shad3.py s1.py s2.py s3.py s4.py
+[file s1.py]
+def foo() -> str:
+    return "bar"
+[file shad1.py]
+def bar() -> str:
+    return 14
+[file s2.py]
+def baz() -> str:
+    return 14
+[file shad2.py]
+def baz() -> int:
+    return 14
+[file s3.py]
+def qux() -> str:
+    return "bar"
+[file shad3.py]
+def foo() -> int:
+    return [42]
+[file s4.py]
+def foo() -> str:
+    return 9
+[out]
+s4.py:2: error: Incompatible return value type (got "int", expected "str")
+s3.py:2: error: Incompatible return value type (got "List[int]", expected "int")
+s1.py:2: error: Incompatible return value type (got "int", expected "str")
+
 [case testConfigWarnUnusedSection1]
 # cmd: mypy foo.py quux.py spam/eggs.py
 # flags: --follow-imports=skip