From 42d4a6fcde4faeb4ed33cd4ea5cad1c0f0038ad6 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Tue, 8 May 2018 10:53:43 -0400 Subject: [PATCH 01/10] Start checking multiple shadow paths. --- mypy/build.py | 24 +++++++++++++++++++++--- mypy/fscache.py | 5 +++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index f6fb27ae74b6..71189f4f8a9b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -655,13 +655,31 @@ def __init__(self, data_dir: str, self.fscache = fscache self.find_module_cache = FindModuleCache(self.fscache) + # a mapping from files to their corresponding shadow files + self.shadow_map = {} # type: Dict[str, str] + # a mapping from incoming files to files that exist in the shadow map + self.shadow_equivalence_map = {} # type: Dict[str, Optional[str]] + def use_fine_grained_cache(self) -> bool: return self.cache_enabled and self.options.use_fine_grained_cache def maybe_swap_for_shadow_path(self, path: str) -> str: - if (self.options.shadow_file and - os.path.samefile(self.options.shadow_file[0], path)): - path = self.options.shadow_file[1] + if not self.options.shadow_file: + return path + + previously_checked = path in self.shadow_equivalence_map + if not previously_checked: + for k in self.shadow_map.keys(): + if self.fscache.samefile(path, k): + self.shadow_equivalence_map[path] = self.shadow_map.get(k) + break + else: + self.shadow_equivalence_map[path] = None + + shadow_file = self.shadow_equivalence_map.get(path) + if shadow_file: + path = shadow_file + return path def get_stat(self, path: str) -> os.stat_result: diff --git a/mypy/fscache.py b/mypy/fscache.py index 00ba3561bbc1..2db3413e7f7d 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -149,6 +149,11 @@ def md5(self, path: str) -> str: self.read(path) return self.hash_cache[path] + def samefile(self, f1: str, f2: str) -> bool: + s1 = self.stat(f1) + s2 = self.stat(f2) + return os.path.samestat(s1, s2) + def copy_os_error(e: OSError) -> OSError: new = OSError(*e.args) From d69b68d3f76a191a693904ccb87aa72d4a1a6ee3 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Wed, 9 May 2018 23:30:59 -0400 Subject: [PATCH 02/10] Allow --shadow-file to take multiple parameters. --- mypy/build.py | 9 +++++++-- mypy/main.py | 2 +- mypy/options.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 71189f4f8a9b..86905673be4f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -655,9 +655,14 @@ def __init__(self, data_dir: str, self.fscache = fscache self.find_module_cache = FindModuleCache(self.fscache) - # a mapping from files to their corresponding shadow files + # a mapping from source files to their corresponding shadow files + # for efficient lookup self.shadow_map = {} # type: Dict[str, str] - # a mapping from incoming files to files that exist in the shadow map + if self.options.shadow_file is not None: + self.shadow_map = {source_file: shadow_file + for (source_file, shadow_file) + in self.options.shadow_file} + # a mapping from typechecked files to files to their possible shadow files self.shadow_equivalence_map = {} # type: Dict[str, Optional[str]] def use_fine_grained_cache(self) -> bool: diff --git a/mypy/main.py b/mypy/main.py index 965361b34c5f..569978750449 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -463,7 +463,7 @@ def add_invertible_flag(flag: str, parser.add_argument('--strict', action='store_true', dest='special-opts:strict', help=strict_help) parser.add_argument('--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'), - dest='shadow_file', + dest='shadow_file', action='append', help='Typecheck SHADOW_FILE in place of SOURCE_FILE.') # hidden options # --debug-cache will disable any cache-related compressions/optimizations, diff --git a/mypy/options.py b/mypy/options.py index 673e88bfe16e..befab1ec48ac 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -185,7 +185,7 @@ def __init__(self) -> None: self.use_builtins_fixtures = False # -- experimental options -- - self.shadow_file = None # type: Optional[Tuple[str, str]] + self.shadow_file = None # type: Optional[List[Tuple[str, str]]] self.show_column_numbers = False # type: bool self.dump_graph = False self.dump_deps = False From 62b79a24bc4df527ede7c4ab8d9dcf3e59eee174 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Wed, 9 May 2018 23:31:17 -0400 Subject: [PATCH 03/10] Add tests for singular shadow file and multiple shadow files. --- test-data/unit/cmdline.test | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 84da97c5bec8..3e7620e5eb5a 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1126,6 +1126,45 @@ follow_imports_for_stubs = True import math math.frobnicate() +[case testShadowFile1] +# cmd: mypy --shadow-file source.py shadow.py source.py +[file source.py] +def foo() -> str: + return "bar" +[file shadow.py] +def bar() -> str: + return 14 +[out] +source.py:2: error: Incompatible return value type (got "int", expected "str") + +[case testShadowFile2] +# cmd: mypy --shadow-file s1.py shad1.py --shadow-file s2.py shad2.py --shadow-file s3.py shad3.py s1.py s2.py s3.py s4.py +[file s1.py] +def foo() -> str: + return "bar" +[file shad1.py] +def bar() -> str: + return 14 +[file s2.py] +def baz() -> str: + return 14 +[file shad2.py] +def baz() -> int: + return 14 +[file s3.py] +def qux() -> str: + return "bar" +[file shad3.py] +def foo() -> int: + return [42] +[file s4.py] +def foo() -> str: + return 9 +[out] +s4.py:2: error: Incompatible return value type (got "int", expected "str") +s3.py:2: error: Incompatible return value type (got "List[int]", expected "int") +s1.py:2: error: Incompatible return value type (got "int", expected "str") + [case testConfigWarnUnusedSection1] # cmd: mypy foo.py quux.py spam/eggs.py # flags: --follow-imports=skip From 28ca8520ee061c34e2308fddffea61b742e9d071 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Wed, 9 May 2018 23:39:24 -0400 Subject: [PATCH 04/10] Add comment to documentation. --- docs/source/command_line.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 5b870eab122e..b0c9dc0e870c 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -468,7 +468,8 @@ Here are some more useful flags: make transformations to a file before type checking without having to change the file in-place. (For example, tooling could use this to display the type of an expression by wrapping it with a call to reveal_type in the shadow - file and then parsing the output.) + file and then parsing the output.) This argument may be specified multiple times + to make mypy substitute multiple different files with their shadow replacements. .. _no-implicit-optional: From 003545a569e207680f46b996b5d61f22a81b3194 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Sat, 12 May 2018 10:31:22 -0400 Subject: [PATCH 05/10] Fix comment. --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 86905673be4f..18fc07e4cb9c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -662,7 +662,7 @@ def __init__(self, data_dir: str, self.shadow_map = {source_file: shadow_file for (source_file, shadow_file) in self.options.shadow_file} - # a mapping from typechecked files to files to their possible shadow files + # a mapping from each file being typechecked to its possible shadow file self.shadow_equivalence_map = {} # type: Dict[str, Optional[str]] def use_fine_grained_cache(self) -> bool: From 605a37dbd87ec15e65d4c1b1e5ef88c70fc501b4 Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Mon, 14 May 2018 22:35:49 -0400 Subject: [PATCH 06/10] Ignore type error because of missing stub until python/typeshed#2053 is merged. --- mypy/fscache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fscache.py b/mypy/fscache.py index 2db3413e7f7d..9b89144e7685 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -152,7 +152,7 @@ def md5(self, path: str) -> str: def samefile(self, f1: str, f2: str) -> bool: s1 = self.stat(f1) s2 = self.stat(f2) - return os.path.samestat(s1, s2) + return os.path.samestat(s1, s2) # type: ignore def copy_os_error(e: OSError) -> OSError: From 34677a9b3a32f559d07b7051134593c8db061aab Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Tue, 15 May 2018 13:28:01 -0400 Subject: [PATCH 07/10] Clean up code. --- mypy/build.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 18fc07e4cb9c..4d3209974a0f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -669,23 +669,19 @@ def use_fine_grained_cache(self) -> bool: return self.cache_enabled and self.options.use_fine_grained_cache def maybe_swap_for_shadow_path(self, path: str) -> str: - if not self.options.shadow_file: + if not self.shadow_map: return path previously_checked = path in self.shadow_equivalence_map if not previously_checked: - for k in self.shadow_map.keys(): - if self.fscache.samefile(path, k): - self.shadow_equivalence_map[path] = self.shadow_map.get(k) + for source, shadow in self.shadow_map.items(): + if self.fscache.samefile(path, source): + self.shadow_equivalence_map[path] = shadow break else: self.shadow_equivalence_map[path] = None - shadow_file = self.shadow_equivalence_map.get(path) - if shadow_file: - path = shadow_file - - return path + return self.shadow_equivalence_map.get(path, path) def get_stat(self, path: str) -> os.stat_result: return self.fscache.stat(self.maybe_swap_for_shadow_path(path)) From 0ec018bea92ea42c0a58bab3c76f3d08b0b46abb Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Tue, 15 May 2018 15:21:24 -0400 Subject: [PATCH 08/10] Rewrite documentation for --shadow-file. --- docs/source/command_line.rst | 21 ++++++++++++++------- mypy/main.py | 3 ++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index b0c9dc0e870c..ba198a1d7900 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -463,13 +463,20 @@ Here are some more useful flags: .. _shadow-file: -- ``--shadow-file SOURCE_FILE SHADOW_FILE`` makes mypy typecheck SHADOW_FILE in - place of SOURCE_FILE. Primarily intended for tooling. Allows tooling to - make transformations to a file before type checking without having to change - the file in-place. (For example, tooling could use this to display the type - of an expression by wrapping it with a call to reveal_type in the shadow - file and then parsing the output.) This argument may be specified multiple times - to make mypy substitute multiple different files with their shadow replacements. +- ``--shadow-file SOURCE_FILE SHADOW_FILE``: when mypy is asked to typecheck + ``SOURCE_FILE``, this makes it read from and typecheck the contents of + ``SHADOW_FILE`` instead. However, diagnostics will continue to refer to + ``SOURCE_FILE``. Specifying this argument multiple times + (``--shadow-file X1 Y1 --shadow-file X2 Y2``) + will allow mypy to perform multiple substitutions. + + This allows tooling to create temporary files with helpful modifications + without having to change the source file in place. For example, suppose we + have a pipeline that adds ``reveal_type`` for certain variables. + This pipeline is run on ``original.py`` to produce ``temp.py``. + Running ``mypy --shadow-file original.py temp.py original.py`` will then + cause mypy to typecheck the contents of ``temp.py`` instead of ``original.py``, + but error messages will still reference ``original.py``. .. _no-implicit-optional: diff --git a/mypy/main.py b/mypy/main.py index 569978750449..f752887ff1f7 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -464,7 +464,8 @@ def add_invertible_flag(flag: str, help=strict_help) parser.add_argument('--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'), dest='shadow_file', action='append', - help='Typecheck SHADOW_FILE in place of SOURCE_FILE.') + help='When encountering SOURCE_FILE, read and typecheck' + 'the contents of SHADOW_FILE instead.') # hidden options # --debug-cache will disable any cache-related compressions/optimizations, # which will make the cache writing process output pretty-printed JSON (which From 73752d7bb2e9c0fabdc21f5b63212b1fd964cacf Mon Sep 17 00:00:00 2001 From: Gareth Tan Date: Wed, 16 May 2018 02:31:04 -0400 Subject: [PATCH 09/10] Simplify in a way that works. --- mypy/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 4d3209974a0f..14cc1d5a2c22 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -681,7 +681,8 @@ def maybe_swap_for_shadow_path(self, path: str) -> str: else: self.shadow_equivalence_map[path] = None - return self.shadow_equivalence_map.get(path, path) + shadow_file = self.shadow_equivalence_map.get(path) + return shadow_file if shadow_file else path def get_stat(self, path: str) -> os.stat_result: return self.fscache.stat(self.maybe_swap_for_shadow_path(path)) From 3af2ed6efb72268f73614704324e53e83c60dc9e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 16 May 2018 07:21:44 -0400 Subject: [PATCH 10/10] Add extra space to help message Avoid run-in words due to line break. (Also switched to "Human quotes".) --- mypy/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index f752887ff1f7..4e1af5518da1 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -464,8 +464,8 @@ def add_invertible_flag(flag: str, help=strict_help) parser.add_argument('--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'), dest='shadow_file', action='append', - help='When encountering SOURCE_FILE, read and typecheck' - 'the contents of SHADOW_FILE instead.') + help="When encountering SOURCE_FILE, read and typecheck " + "the contents of SHADOW_FILE instead.") # hidden options # --debug-cache will disable any cache-related compressions/optimizations, # which will make the cache writing process output pretty-printed JSON (which