Use Tokens from parsed type annotation or parsed source (#11740)

## Summary

This PR fixes a bug where the checker would require the tokens for an
invalid offset w.r.t. the source code.

Taking the source code from the linked issue as an example:
```py
relese_version :"0.0is 64"
```

Now, this isn't really a valid type annotation but that's what this PR
is fixing. Regardless of whether it's valid or not, Ruff shouldn't
panic.

The checker would visit the parsed type annotation (`0.0is 64`) and try
to detect any violations. Certain rule logic requests the tokens for the
same but it would fail because the lexer would only have the `String`
token considering original source code. This worked before because the
lexer was invoked again for each rule logic.

The solution is to store the parsed type annotation on the checker if
it's in a typing context and use the tokens from that instead if it's
available. This is enforced by creating a new API on the checker to get
the tokens.

But, this means that there are two ways to get the tokens via the
checker API. I want to restrict this in a follow-up PR (#11741) to only
expose `tokens` and `comment_ranges` as methods and restrict access to
the parsed source code.

fixes: #11736 

## Test Plan

- [x] Add a test case for `F632` rule and update the snapshot
- [x] Check all affected rules
- [x] No ecosystem changes

Author: dhruvmanila

crates/ruff_linter/resources/test/fixtures/pyflakes/F632.py

@@ -29,3 +29,6 @@
 # Regression test for
 if values[1is not None ] is not '-':
     pass
+
+# Regression test for https://github.com/astral-sh/ruff/issues/11736
+variable: "123 is not y"

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP012.py

@@ -80,3 +80,7 @@
 # Regression test for: https://github.com/astral-sh/ruff/issues/7455#issuecomment-1722459882
 def _match_ignore(line):
 	input=stdin and'\n'.encode()or None
+
+# Not a valid type annotation but this test shouldn't result in a panic.
+# Refer: https://github.com/astral-sh/ruff/issues/11736
+x: '"foo".encode("utf-8")'

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP015_1.py

@@ -0,0 +1,4 @@
+# Not a valid type annotation but this test shouldn't result in a panic.
+# Refer: https://github.com/astral-sh/ruff/issues/11736
+x: 'open("foo", "r")'
+

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP031_0.py

@@ -125,3 +125,7 @@
 'Hello %s' % bar.baz
 
 'Hello %s' % bar['bop']
+
+# Not a valid type annotation but this test shouldn't result in a panic.
+# Refer: https://github.com/astral-sh/ruff/issues/11736
+x: "'%s + %s' % (1, 2)"

crates/ruff_linter/resources/test/fixtures/pyupgrade/UP032_0.py

@@ -265,3 +265,7 @@ async def c():
 
 # The call should be removed, but the string itself should remain.
 "".format(self.project)
+
+# Not a valid type annotation but this test shouldn't result in a panic.
+# Refer: https://github.com/astral-sh/ruff/issues/11736
+x: "'{} + {}'.format(x, y)"

crates/ruff_linter/src/checkers/ast/mod.rs

@@ -49,7 +49,7 @@ use ruff_python_ast::{helpers, str, visitor, PySourceType};
 use ruff_python_codegen::{Generator, Stylist};
 use ruff_python_index::Indexer;
 use ruff_python_parser::typing::{parse_type_annotation, AnnotationKind};
-use ruff_python_parser::Parsed;
+use ruff_python_parser::{Parsed, Tokens};
 use ruff_python_semantic::all::{DunderAllDefinition, DunderAllFlags};
 use ruff_python_semantic::analyze::{imports, typing};
 use ruff_python_semantic::{
@@ -176,8 +176,10 @@ impl ExpectedDocstringKind {
 }
 
 pub(crate) struct Checker<'a> {
-    /// The parsed [`Parsed`].
+    /// The [`Parsed`] output for the source code.
     parsed: &'a Parsed<ModModule>,
+    /// The [`Parsed`] output for the type annotation the checker is currently in.
+    parsed_type_annotation: Option<&'a Parsed<ModExpression>>,
     /// The [`Path`] to the file under analysis.
     path: &'a Path,
     /// The [`Path`] to the package containing the current file.
@@ -243,6 +245,7 @@ impl<'a> Checker<'a> {
     ) -> Checker<'a> {
         Checker {
             parsed,
+            parsed_type_annotation: None,
             settings,
             noqa_line_for,
             noqa,
@@ -328,6 +331,16 @@ impl<'a> Checker<'a> {
         self.parsed
     }
 
+    /// Returns the [`Tokens`] for the parsed type annotation if the checker is in a typing context
+    /// or the parsed source code.
+    pub(crate) fn tokens(&self) -> &'a Tokens {
+        if let Some(parsed_type_annotation) = self.parsed_type_annotation {
+            parsed_type_annotation.tokens()
+        } else {
+            self.parsed.tokens()
+        }
+    }
+
     /// The [`Locator`] for the current file, which enables extraction of source code from byte
     /// offsets.
     pub(crate) const fn locator(&self) -> &'a Locator<'a> {
@@ -2160,6 +2173,7 @@ impl<'a> Checker<'a> {
                     parse_type_annotation(string_expr, self.locator.contents())
                 {
                     let parsed_annotation = allocator.alloc(parsed_annotation);
+                    self.parsed_type_annotation = Some(parsed_annotation);
 
                     let annotation = string_expr.value.to_str();
                     let range = string_expr.range();
@@ -2187,6 +2201,7 @@ impl<'a> Checker<'a> {
                     self.semantic.flags |=
                         SemanticModelFlags::TYPE_DEFINITION | type_definition_flag;
                     self.visit_expr(parsed_annotation.expr());
+                    self.parsed_type_annotation = None;
                 } else {
                     if self.enabled(Rule::ForwardAnnotationSyntaxError) {
                         self.diagnostics.push(Diagnostic::new(

crates/ruff_linter/src/rules/pyflakes/rules/invalid_literal_comparisons.rs

@@ -96,7 +96,7 @@ pub(crate) fn invalid_literal_comparison(
         {
             let mut diagnostic = Diagnostic::new(IsLiteral { cmp_op: op.into() }, expr.range());
             if lazy_located.is_none() {
-                lazy_located = Some(locate_cmp_ops(expr, checker.parsed().tokens()));
+                lazy_located = Some(locate_cmp_ops(expr, checker.tokens()));
             }
             if let Some(located_op) = lazy_located.as_ref().and_then(|located| located.get(index)) {
                 assert_eq!(located_op.op, *op);

crates/ruff_linter/src/rules/pyflakes/rules/unused_variable.rs

@@ -170,11 +170,10 @@ fn remove_unused_variable(binding: &Binding, checker: &Checker) -> Option<Fix> {
                     )
                     .unwrap_or(target.range())
                     .start();
-                    let end =
-                        match_token_after(checker.parsed().tokens(), target.end(), |token| {
-                            token == TokenKind::Equal
-                        })?
-                        .start();
+                    let end = match_token_after(checker.tokens(), target.end(), |token| {
+                        token == TokenKind::Equal
+                    })?
+                    .start();
                     let edit = Edit::deletion(start, end);
                     Some(Fix::unsafe_edit(edit))
                 } else {
@@ -206,10 +205,9 @@ fn remove_unused_variable(binding: &Binding, checker: &Checker) -> Option<Fix> {
                 // If the expression is complex (`x = foo()`), remove the assignment,
                 // but preserve the right-hand side.
                 let start = statement.start();
-                let end = match_token_after(checker.parsed().tokens(), start, |token| {
-                    token == TokenKind::Equal
-                })?
-                .start();
+                let end =
+                    match_token_after(checker.tokens(), start, |token| token == TokenKind::Equal)?
+                        .start();
                 let edit = Edit::deletion(start, end);
                 Some(Fix::unsafe_edit(edit))
             } else {
@@ -228,19 +226,17 @@ fn remove_unused_variable(binding: &Binding, checker: &Checker) -> Option<Fix> {
             if let Some(optional_vars) = &item.optional_vars {
                 if optional_vars.range() == binding.range() {
                     // Find the first token before the `as` keyword.
-                    let start = match_token_before(
-                        checker.parsed().tokens(),
-                        item.context_expr.start(),
-                        |token| token == TokenKind::As,
-                    )?
-                    .end();
+                    let start =
+                        match_token_before(checker.tokens(), item.context_expr.start(), |token| {
+                            token == TokenKind::As
+                        })?
+                        .end();
 
                     // Find the first colon, comma, or closing bracket after the `as` keyword.
-                    let end =
-                        match_token_or_closing_brace(checker.parsed().tokens(), start, |token| {
-                            token == TokenKind::Colon || token == TokenKind::Comma
-                        })?
-                        .start();
+                    let end = match_token_or_closing_brace(checker.tokens(), start, |token| {
+                        token == TokenKind::Colon || token == TokenKind::Comma
+                    })?
+                    .start();
 
                     let edit = Edit::deletion(start, end);
                     return Some(Fix::unsafe_edit(edit));

crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F632_F632.py.snap

@@ -203,6 +203,8 @@ F632.py:30:4: F632 [*] Use `!=` to compare constant literals
 30    |-if values[1is not None ] is not '-':
    30 |+if values[1is not None ] != '-':
 31 31 |     pass
+32 32 | 
+33 33 | # Regression test for https://github.com/astral-sh/ruff/issues/11736
 
 F632.py:30:11: F632 [*] Use `!=` to compare constant literals
    |
@@ -220,5 +222,20 @@ F632.py:30:11: F632 [*] Use `!=` to compare constant literals
 30    |-if values[1is not None ] is not '-':
    30 |+if values[1!= None ] is not '-':
 31 31 |     pass
+32 32 | 
+33 33 | # Regression test for https://github.com/astral-sh/ruff/issues/11736
 
+F632.py:34:12: F632 [*] Use `!=` to compare constant literals
+   |
+33 | # Regression test for https://github.com/astral-sh/ruff/issues/11736
+34 | variable: "123 is not y"
+   |            ^^^^^^^^^^^^ F632
+   |
+   = help: Replace `is not` with `!=`
 
+ℹ Safe fix
+31 31 |     pass
+32 32 | 
+33 33 | # Regression test for https://github.com/astral-sh/ruff/issues/11736
+34    |-variable: "123 is not y"
+   34 |+variable: "123 != y"

crates/ruff_linter/src/rules/pyupgrade/mod.rs

@@ -58,6 +58,7 @@ mod tests {
     #[test_case(Rule::QuotedAnnotation, Path::new("UP037_0.py"))]
     #[test_case(Rule::QuotedAnnotation, Path::new("UP037_1.py"))]
     #[test_case(Rule::RedundantOpenModes, Path::new("UP015.py"))]
+    #[test_case(Rule::RedundantOpenModes, Path::new("UP015_1.py"))]
     #[test_case(Rule::ReplaceStdoutStderr, Path::new("UP022.py"))]
     #[test_case(Rule::ReplaceUniversalNewlines, Path::new("UP021.py"))]
     #[test_case(Rule::SuperCallWithParameters, Path::new("UP008.py"))]