From 652044ccca840dc07dd985f566d1e9609eacd194 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 4 Oct 2024 07:43:55 -0400 Subject: [PATCH 01/23] hack to support "mypy: ignore" comments until the in-built compile function changes to allow us to detect it otherwise. --- mypy/fastparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index bbbe2184738c..c66576341b23 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -138,6 +138,7 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: + source = source.replace("# mypy: ignore", "# type: ignore") # hack to support "mypy: ignore" comments until the in-built compile function changes to allow us to detect it otherwise. return ast3.parse( source, filename, From af861258ef82cabb6a8625c4030f1360a3f171db Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 4 Oct 2024 07:49:00 -0400 Subject: [PATCH 02/23] Use re.sub instead of str.replace, so as to match type ignore behavior better. --- mypy/fastparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c66576341b23..15e8f819472e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -138,7 +138,7 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: - source = source.replace("# mypy: ignore", "# type: ignore") # hack to support "mypy: ignore" comments until the in-built compile function changes to allow us to detect it otherwise. + source = re.sub(r"#\s*mypy:\s*ignore", "# type: ignore", source) # hack to support "mypy: ignore" comments until the in-built compile function changes to allow us to detect it otherwise. return ast3.parse( source, filename, From 7831c702fb01b5f6d978b7a2fdbe73b64af12c7c Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Fri, 4 Oct 2024 08:47:55 -0400 Subject: [PATCH 03/23] Update fastparse.py: exclude start of line --- mypy/fastparse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 15e8f819472e..0cdf1df2517b 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -138,7 +138,9 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: - source = re.sub(r"#\s*mypy:\s*ignore", "# type: ignore", source) # hack to support "mypy: ignore" comments until the in-built compile function changes to allow us to detect it otherwise. + # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: + # (does not apply at the start of the line to avoid conflicting with mypy file configuration comments https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) + source = re.sub(r"(? Date: Fri, 4 Oct 2024 06:04:46 -0700 Subject: [PATCH 04/23] make the re.sub take str or bytes, like the function signature --- mypy/fastparse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 0cdf1df2517b..7ebc248398d8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -140,7 +140,10 @@ def ast3_parse( ) -> AST: # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: # (does not apply at the start of the line to avoid conflicting with mypy file configuration comments https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) - source = re.sub(r"(? Date: Fri, 4 Oct 2024 07:19:22 -0700 Subject: [PATCH 05/23] use a (?![-_]) to further ignore inline-configs --- mypy/fastparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7ebc248398d8..f80fa22a125a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -141,9 +141,9 @@ def ast3_parse( # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: # (does not apply at the start of the line to avoid conflicting with mypy file configuration comments https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) if isinstance(source, str): - source = re.sub(r"(? Date: Sat, 5 Oct 2024 05:21:40 -0700 Subject: [PATCH 06/23] use python's tokenize, in order to limit the replacement to only comments note also that the 'not start of line' constraint had to be removed from the regex, because now each comment is encountered individually and thus they are at the start --- mypy/fastparse.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index f80fa22a125a..5f281e1ecd17 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,8 +1,10 @@ from __future__ import annotations import copy +import io import re import sys +import tokenize import warnings from typing import Any, Callable, Final, List, Optional, Sequence, TypeVar, Union, cast from typing_extensions import Literal, overload @@ -138,19 +140,26 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: + """This function is just a convenience wrapper around ast.parse, with default flags useful to Mypy. + It also incorporates a hack to accomodate `# mypy: ignore` comments, which are treated by mypy as `# type: ignore` comments.""" # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: - # (does not apply at the start of the line to avoid conflicting with mypy file configuration comments https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) + # (Note: completely distinct from https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) + + # We make the substitution in comments, and to find those comments we use Python's `tokenize`. + # https://docs.python.org/3/library/tokenize.html has a big red **Warning:** + # Note that the functions in this module are only designed to parse syntactically valid Python code (code that does not raise when parsed using ast.parse()). The behavior of the functions in this module is **undefined** when providing invalid Python code and it can change at any point. + # So, we cannot rely on roundtrip behavior in tokenize iff ast.parse would throw when given `source`. + # The simplest way to deal with that is just to call ast.parse twice, once before and once after. So, we do that. + p = lambda: ast3.parse(source, filename, mode, type_comments=True, feature_version=feature_version) + p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). if isinstance(source, str): - source = re.sub(r"(? Date: Sat, 5 Oct 2024 12:22:10 +0000 Subject: [PATCH 07/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 5f281e1ecd17..cf3fa569c3db 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -141,7 +141,8 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: """This function is just a convenience wrapper around ast.parse, with default flags useful to Mypy. - It also incorporates a hack to accomodate `# mypy: ignore` comments, which are treated by mypy as `# type: ignore` comments.""" + It also incorporates a hack to accomodate `# mypy: ignore` comments, which are treated by mypy as `# type: ignore` comments. + """ # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: # (Note: completely distinct from https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) @@ -150,15 +151,19 @@ def ast3_parse( # Note that the functions in this module are only designed to parse syntactically valid Python code (code that does not raise when parsed using ast.parse()). The behavior of the functions in this module is **undefined** when providing invalid Python code and it can change at any point. # So, we cannot rely on roundtrip behavior in tokenize iff ast.parse would throw when given `source`. # The simplest way to deal with that is just to call ast.parse twice, once before and once after. So, we do that. - p = lambda: ast3.parse(source, filename, mode, type_comments=True, feature_version=feature_version) - p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). + p = lambda: ast3.parse( + source, filename, mode, type_comments=True, feature_version=feature_version + ) + p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). if isinstance(source, str): tokens = tokenize.generate_tokens(io.StringIO(source).readline) to_find, to_replace = r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore" else: tokens = tokenize.tokenize(io.BytesIO(source).readline) to_find, to_replace = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" - source = tokenize.untokenize((t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens) + source = tokenize.untokenize( + (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens + ) return p() From 9d02634b578bf0d172396966c8c1e776cd291fc9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 5 Oct 2024 05:28:47 -0700 Subject: [PATCH 08/23] appease the typechecker --- mypy/fastparse.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index cf3fa569c3db..c39bcd8519d9 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -151,19 +151,23 @@ def ast3_parse( # Note that the functions in this module are only designed to parse syntactically valid Python code (code that does not raise when parsed using ast.parse()). The behavior of the functions in this module is **undefined** when providing invalid Python code and it can change at any point. # So, we cannot rely on roundtrip behavior in tokenize iff ast.parse would throw when given `source`. # The simplest way to deal with that is just to call ast.parse twice, once before and once after. So, we do that. - p = lambda: ast3.parse( - source, filename, mode, type_comments=True, feature_version=feature_version - ) + def p() -> AST: + return ast3.parse( + source, filename, mode, type_comments=True, feature_version=feature_version + ) p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). if isinstance(source, str): tokens = tokenize.generate_tokens(io.StringIO(source).readline) to_find, to_replace = r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore" + source = tokenize.untokenize( + (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens + ) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) to_find, to_replace = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" - source = tokenize.untokenize( - (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens - ) + source = tokenize.untokenize( + (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens + ) return p() From 8868c63b0cfbd889fbf4a44c211e0b6dc2d303a4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Oct 2024 12:29:46 +0000 Subject: [PATCH 09/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c39bcd8519d9..ecd4b63b977e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -155,18 +155,21 @@ def p() -> AST: return ast3.parse( source, filename, mode, type_comments=True, feature_version=feature_version ) + p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). if isinstance(source, str): tokens = tokenize.generate_tokens(io.StringIO(source).readline) to_find, to_replace = r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore" source = tokenize.untokenize( - (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens + (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) + for t, s, *_ in tokens ) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) to_find, to_replace = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" source = tokenize.untokenize( - (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens + (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) + for t, s, *_ in tokens ) return p() From 6feee3ca170d073d7f61290e58900f13b63dc826 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 5 Oct 2024 05:34:12 -0700 Subject: [PATCH 10/23] appease shadow rule I guess --- mypy/fastparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index ecd4b63b977e..2e6e9a990289 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -166,9 +166,9 @@ def p() -> AST: ) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) - to_find, to_replace = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" + to_find_bytes, to_replace_bytes = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" source = tokenize.untokenize( - (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) + (t, re.sub(to_find_bytes, to_replace_bytes, s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens ) return p() From 6bcdd3f7da6f2293edf8b2c6f1a471e8b2823d36 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sat, 5 Oct 2024 05:54:20 -0700 Subject: [PATCH 11/23] I guess it's always a string in the comprehension, actually --- mypy/fastparse.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 2e6e9a990289..7542bb68503e 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -159,18 +159,12 @@ def p() -> AST: p() # Call to assure syntactic validity (will throw an exception otherwise, exiting this function). if isinstance(source, str): tokens = tokenize.generate_tokens(io.StringIO(source).readline) - to_find, to_replace = r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore" - source = tokenize.untokenize( - (t, re.sub(to_find, to_replace, s) if t == tokenize.COMMENT else s) - for t, s, *_ in tokens - ) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) - to_find_bytes, to_replace_bytes = rb"#\s*mypy:\s*ignore(?![-_])", b"# type: ignore" - source = tokenize.untokenize( - (t, re.sub(to_find_bytes, to_replace_bytes, s) if t == tokenize.COMMENT else s) + source = tokenize.untokenize( + (t, re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) if t == tokenize.COMMENT else s) for t, s, *_ in tokens - ) + ) return p() From 38d7cbcd5eb6f49f44db492dadce377c3fb430ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 5 Oct 2024 12:54:47 +0000 Subject: [PATCH 12/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7542bb68503e..ca4b2a5e2932 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -162,8 +162,15 @@ def p() -> AST: else: tokens = tokenize.tokenize(io.BytesIO(source).readline) source = tokenize.untokenize( - (t, re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) if t == tokenize.COMMENT else s) - for t, s, *_ in tokens + ( + t, + ( + re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) + if t == tokenize.COMMENT + else s + ), + ) + for t, s, *_ in tokens ) return p() From ef13623265fbb194e99d99e177162420673e2a8e Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 6 Oct 2024 17:27:05 -0700 Subject: [PATCH 13/23] implement workaround for \n{{ roundtripping bug cf https://github.com/python/cpython/issues/125008 --- mypy/fastparse.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index ca4b2a5e2932..df6a07b10c5b 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -161,16 +161,14 @@ def p() -> AST: tokens = tokenize.generate_tokens(io.StringIO(source).readline) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) + # We do a workaround for a roundtripping error https://github.com/python/cpython/issues/125008 + # An error that was introduced in 3.12.3 (https://github.com/python/cpython/blob/v3.12.3/Lib/tokenize.py#L205) and fixed in 3.12.8 (not out yet at time of writing but it's probably https://github.com/python/cpython/blob/v3.12.8/Lib/tokenize.py#L203 or thereabouts). + # Luckily, it was caught before the first official (non-rc) release of python 3.13. + is_defective_version = (3, 12, 3) <= sys.version_info[:3] <= (3, 12, 7) + # Could the workaround ever come back to bite us? I confess I don't know enough about FSTRING_MIDDLE to say no for certain, even though I have tried to examine all relevant cases. source = tokenize.untokenize( - ( - t, - ( - re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) - if t == tokenize.COMMENT - else s - ), - ) - for t, s, *_ in tokens + (t, re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) if t == tokenize.COMMENT else s+'{' if is_defective_version and t == tokenize.FSTRING_MIDDLE and s.startswith('\\') and s.endswith('{') else s) + for t, s, *_ in tokens ) return p() From 62f034d900b984e162920f63563f769d53acf85c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 00:31:34 +0000 Subject: [PATCH 14/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index df6a07b10c5b..2e95a17461c0 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -167,8 +167,22 @@ def p() -> AST: is_defective_version = (3, 12, 3) <= sys.version_info[:3] <= (3, 12, 7) # Could the workaround ever come back to bite us? I confess I don't know enough about FSTRING_MIDDLE to say no for certain, even though I have tried to examine all relevant cases. source = tokenize.untokenize( - (t, re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) if t == tokenize.COMMENT else s+'{' if is_defective_version and t == tokenize.FSTRING_MIDDLE and s.startswith('\\') and s.endswith('{') else s) - for t, s, *_ in tokens + ( + t, + ( + re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) + if t == tokenize.COMMENT + else ( + s + "{" + if is_defective_version + and t == tokenize.FSTRING_MIDDLE + and s.startswith("\\") + and s.endswith("{") + else s + ) + ), + ) + for t, s, *_ in tokens ) return p() From 392a37c77fc06d763044fe8288dc44032bc2b9a4 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 6 Oct 2024 18:32:54 -0700 Subject: [PATCH 15/23] refix {{ roundtrip, fix hasattr static check --- mypy/fastparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 2e95a17461c0..70b7855dd7ec 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -173,8 +173,9 @@ def p() -> AST: re.sub(r"#\s*mypy:\s*ignore(?![-_])", "# type: ignore", s) if t == tokenize.COMMENT else ( - s + "{" + s[:-1] if is_defective_version + and hasattr(tokenize, "FSTRING_MIDDLE") #technically redundant as all the defective versions have this, but we'd like to appease the typechecker here and t == tokenize.FSTRING_MIDDLE and s.startswith("\\") and s.endswith("{") From 0bb08864289e0f8c1256c26ae1d9359d01b8be1e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 01:33:25 +0000 Subject: [PATCH 16/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 70b7855dd7ec..a8f686c97a4a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -175,7 +175,9 @@ def p() -> AST: else ( s[:-1] if is_defective_version - and hasattr(tokenize, "FSTRING_MIDDLE") #technically redundant as all the defective versions have this, but we'd like to appease the typechecker here + and hasattr( + tokenize, "FSTRING_MIDDLE" + ) # technically redundant as all the defective versions have this, but we'd like to appease the typechecker here and t == tokenize.FSTRING_MIDDLE and s.startswith("\\") and s.endswith("{") From 69b0ce77a38957aed81f58aeed25ecfe68e38beb Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Sun, 6 Oct 2024 19:49:24 -0700 Subject: [PATCH 17/23] fix whitespace roundtripping? --- mypy/fastparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index a8f686c97a4a..ecaac08ebcb9 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -175,15 +175,15 @@ def p() -> AST: else ( s[:-1] if is_defective_version - and hasattr( - tokenize, "FSTRING_MIDDLE" - ) # technically redundant as all the defective versions have this, but we'd like to appease the typechecker here + # Technically redundant hasattr check as all the defective versions are versions that have this attribute, but we'd like to appease the typechecker here: + and hasattr(tokenize, "FSTRING_MIDDLE") and t == tokenize.FSTRING_MIDDLE and s.startswith("\\") and s.endswith("{") else s ) ), + *_ #this improves whitespace roundtripping (possibly always making it perfect, for this usecase?) ) for t, s, *_ in tokens ) From 8d07bb6a03f50715be7f282733ebd5ab21de4f5f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 02:55:02 +0000 Subject: [PATCH 18/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index ecaac08ebcb9..f4710ebec134 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -183,7 +183,7 @@ def p() -> AST: else s ) ), - *_ #this improves whitespace roundtripping (possibly always making it perfect, for this usecase?) + *_, # this improves whitespace roundtripping (possibly always making it perfect, for this usecase?) ) for t, s, *_ in tokens ) From 23afc30e00aa3da2aff5fcfe15cdb789276624b9 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 13 Jan 2025 02:58:38 -0800 Subject: [PATCH 19/23] oh right; I do need io --- mypy/fastparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 66bc37f19c08..8e81596ea821 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import re import sys import tokenize From c0e01712b468fb350333da7226cb47ee584ee66e Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 13 Jan 2025 03:30:17 -0800 Subject: [PATCH 20/23] try adding one test case --- test-data/unit/check-ignore.test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-data/unit/check-ignore.test b/test-data/unit/check-ignore.test index fa451f373e70..878c06d6efb6 100644 --- a/test-data/unit/check-ignore.test +++ b/test-data/unit/check-ignore.test @@ -3,6 +3,11 @@ x = 1 x() # type: ignore x() # E: "int" not callable +[case testMypyIgnoreTypeError] +x = 1 +x() # mypy: ignore +x() # E: "int" not callable + [case testIgnoreUndefinedName] x = 1 y # type: ignore From 069acb94a112e092a825fae28a4156594b1829f1 Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 13 Jan 2025 05:28:01 -0800 Subject: [PATCH 21/23] comments and documentation --- CHANGELOG.md | 8 ++++++++ docs/source/common_issues.rst | 26 ++++++++++++++++++-------- mypy/fastparse.py | 21 ++++++++++++--------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5260104f3fe..ee646c02750c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,14 @@ may necessitate changing the location of a `# type: ignore` comment. Contributed by Shantanu Jain (PR [18392](https://github.com/python/mypy/pull/18392), PR [18397](https://github.com/python/mypy/pull/18397)). +### `mypy: ignore` + +`mypy: ignore` comments are now honored by mypy exactly as though they said `type: ignore`. +This allows one to suppress type errors from mypy in particular, without suppressing other type checkers, +which can be desirable if mypy has a bug and other type checkers one runs on the same code do not. + +Contributed by Wyatt S Carpenter (PR [17875](https://github.com/python/mypy/pull/17875)). + ## Mypy 1.14 We’ve just uploaded mypy 1.14 to the Python Package Index ([PyPI](https://pypi.org/project/mypy/)). diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 7165955e67d3..473b8a19c9e3 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -148,14 +148,6 @@ error: The second line is now fine, since the ignore comment causes the name ``frobnicate`` to get an implicit ``Any`` type. -.. note:: - - You can use the form ``# type: ignore[]`` to only ignore - specific errors on the line. This way you are less likely to - silence unexpected errors that are not safe to ignore, and this - will also document what the purpose of the comment is. See - :ref:`error-codes` for more information. - .. note:: The ``# type: ignore`` comment will only assign the implicit ``Any`` @@ -163,6 +155,24 @@ The second line is now fine, since the ignore comment causes the name if we did have a stub available for ``frobnicate`` then mypy would ignore the ``# type: ignore`` comment and typecheck the stub as usual. +You can use the form ``# type: ignore[]`` to only ignore +specific errors on the line. This way you are less likely to +silence unexpected errors that are not safe to ignore, and this +will also document what the purpose of the comment is. See +:ref:`error-codes` for more information. + +Instead of ``# type: ignore``, you may also use the mypy-specific +``# mypy: ignore``, to tell only mypy — and not other type checkers — to +ignore that line. This is mostly useful if mypy has a bug that produces +known-spurious error messages. You may also use the ``# mypy: ignore[]`` +form to only ignore specific errors. Indeed, it is often wisest to use +``# mypy: ignore[]``, as this minimizes your chances of accidentally +suppressing other errors that you don't really want to ignore! + +(``# mypy: ignore`` comments are per-line and should not be confused with +:ref:`inline-configuration` comments, which are per-file configuration directives +that just happen to be formatted a similar way.) + Another option is to explicitly annotate values with type ``Any`` -- mypy will let you perform arbitrary operations on ``Any`` values. Sometimes there is no more precise type you can use for a diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 8e81596ea821..5bd8b60a3b2c 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -140,16 +140,18 @@ def ast3_parse( source: str | bytes, filename: str, mode: str, feature_version: int = PY_MINOR_VERSION ) -> AST: """This function is just a convenience wrapper around ast.parse, with default flags useful to Mypy. - It also incorporates a hack to accomodate `# mypy: ignore` comments, which are treated by mypy as `# type: ignore` comments. + It also handles `# mypy: ignore` comments, by turning them into `# type: ignore` comments. """ # Hack to support "mypy: ignore" comments until the builtin compile function changes to allow us to detect it otherwise: - # (Note: completely distinct from https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase) + # (Note: completely distinct from https://mypy.readthedocs.io/en/stable/inline_config.html ; see also, util.get_mypy_comments in this codebase.) # We make the substitution in comments, and to find those comments we use Python's `tokenize`. # https://docs.python.org/3/library/tokenize.html has a big red **Warning:** - # Note that the functions in this module are only designed to parse syntactically valid Python code (code that does not raise when parsed using ast.parse()). The behavior of the functions in this module is **undefined** when providing invalid Python code and it can change at any point. + # Note that the functions in this module are only designed to parse syntactically valid Python + # code (code that does not raise when parsed using ast.parse()). The behavior of the functions + # in this module is **undefined** when providing invalid Python code and it can change at any point. # So, we cannot rely on roundtrip behavior in tokenize iff ast.parse would throw when given `source`. - # The simplest way to deal with that is just to call ast.parse twice, once before and once after. So, we do that. + # The simplest way to deal with that is just to call ast.parse twice, once before and once after! def p() -> AST: return ast3.parse( source, filename, mode, type_comments=True, feature_version=feature_version @@ -160,11 +162,12 @@ def p() -> AST: tokens = tokenize.generate_tokens(io.StringIO(source).readline) else: tokens = tokenize.tokenize(io.BytesIO(source).readline) - # We do a workaround for a roundtripping error https://github.com/python/cpython/issues/125008 - # An error that was introduced in 3.12.3 (https://github.com/python/cpython/blob/v3.12.3/Lib/tokenize.py#L205) and fixed in 3.12.8 (not out yet at time of writing but it's probably https://github.com/python/cpython/blob/v3.12.8/Lib/tokenize.py#L203 or thereabouts). + # We do a workaround for a roundtripping error (https://github.com/python/cpython/issues/125008) + # that was introduced in 3.12.3 (https://github.com/python/cpython/blob/v3.12.3/Lib/tokenize.py#L205) + # and fixed in 3.12.8 (https://github.com/python/cpython/blob/v3.12.8/Lib/tokenize.py#L205). # Luckily, it was caught before the first official (non-rc) release of python 3.13. is_defective_version = (3, 12, 3) <= sys.version_info[:3] <= (3, 12, 7) - # Could the workaround ever come back to bite us? I confess I don't know enough about FSTRING_MIDDLE to say no for certain, even though I have tried to examine all relevant cases. + # This is a gnarly list comprehension, but that's basically just how untokenize is supposed to work. source = tokenize.untokenize( ( t, @@ -182,11 +185,11 @@ def p() -> AST: else s ) ), - *_, # this improves whitespace roundtripping (possibly always making it perfect, for this usecase?) + *_, # Including this bit improves whitespace roundtripping (possibly always making it perfect). ) for t, s, *_ in tokens ) - return p() + return p() # `source` has changed, so this result is different than the first call. NamedExpr = ast3.NamedExpr From 461d6e93da5abe02549172254696ae9a3f170d99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:28:51 +0000 Subject: [PATCH 22/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypy/fastparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 5bd8b60a3b2c..ffc5c464741f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -189,7 +189,7 @@ def p() -> AST: ) for t, s, *_ in tokens ) - return p() # `source` has changed, so this result is different than the first call. + return p() # `source` has changed, so this result is different than the first call. NamedExpr = ast3.NamedExpr From f4990bee8b8fca6df72ea22a40a97afb4190e6ca Mon Sep 17 00:00:00 2001 From: wyattscarpenter Date: Mon, 13 Jan 2025 05:55:12 -0800 Subject: [PATCH 23/23] fix undefined label: 'inline-configuration' [ref.ref] --- docs/source/common_issues.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index 473b8a19c9e3..eb7097f60fad 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -170,7 +170,7 @@ form to only ignore specific errors. Indeed, it is often wisest to use suppressing other errors that you don't really want to ignore! (``# mypy: ignore`` comments are per-line and should not be confused with -:ref:`inline-configuration` comments, which are per-file configuration directives +:ref:`inline-config` comments, which are per-file configuration directives that just happen to be formatted a similar way.) Another option is to explicitly annotate values with type ``Any`` --