Merge branch 'main' into cjm/cfg4

* main:
  CI: add job to run tests under minimum supported rust version (msrv) (#11737)
  Lexer should consider BOM for the start offset (#11732)
  Use cursor offset for lexer checkpoint (#11734)
  red-knot: Change `resolve_global_symbol` to take `Module` as an argument (#11723)
  red-knot: Use `parse_unchecked` to get all parse errors (#11725)
  Respect per-file ignores for blanket and redirected noqa rules (#11728)
  [`pygrep_hooks`] Check blanket ignores via file-level pragmas (`PGH004`) (#11540)

Author: carljm

.github/workflows/ci.yaml

@@ -212,6 +212,38 @@ jobs:
       - name: "Build"
         run: cargo build --release --locked
 
+  cargo-build-msrv:
+    name: "cargo build (msrv)"
+    runs-on: ubuntu-latest
+    needs: determine_changes
+    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - uses: SebRollen/[email protected]
+        id: msrv
+        with:
+          file: "Cargo.toml"
+          field: "workspace.package.rust-version"
+      - name: "Install Rust toolchain"
+        run: rustup default ${{ steps.msrv.outputs.value }}
+      - name: "Install mold"
+        uses: rui314/setup-mold@v1
+      - name: "Install cargo nextest"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+      - name: "Install cargo insta"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-insta
+      - uses: Swatinem/rust-cache@v2
+      - name: "Run tests"
+        shell: bash
+        env:
+          NEXTEST_PROFILE: "ci"
+        run: cargo +${{ steps.msrv.outputs.value }} insta test --all-features --unreferenced reject --test-runner nextest
+
   cargo-fuzz:
     name: "cargo fuzz"
     runs-on: ubuntu-latest

Cargo.lock

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-rust-version = "1.71"
+rust-version = "1.74"
 homepage = "https://docs.astral.sh/ruff"
 documentation = "https://docs.astral.sh/ruff"
 repository = "https://github.com/astral-sh/ruff"

Cargo.toml

@@ -35,7 +35,6 @@ tracing-subscriber = { workspace = true }
 tracing-tree = { workspace = true }
 
 [dev-dependencies]
-textwrap = { version = "0.16.1" }
 tempfile = { workspace = true }
 
 [lints]

crates/red_knot/Cargo.toml

@@ -5,12 +5,13 @@ use std::time::Duration;
 
 use ruff_python_ast::visitor::Visitor;
 use ruff_python_ast::{ModModule, StringLiteral};
+use ruff_python_parser::Parsed;
 
 use crate::cache::KeyValueCache;
 use crate::db::{LintDb, LintJar, QueryResult};
 use crate::files::FileId;
-use crate::module::ModuleName;
-use crate::parse::{parse, Parsed};
+use crate::module::{resolve_module, ModuleName};
+use crate::parse::parse;
 use crate::source::{source_text, Source};
 use crate::symbols::{
     resolve_global_symbol, symbol_table, Definition, GlobalSymbolId, SymbolId, SymbolTable,
@@ -40,7 +41,7 @@ pub(crate) fn lint_syntax(db: &dyn LintDb, file_id: FileId) -> QueryResult<Diagn
         let parsed = parse(db.upcast(), *file_id)?;
 
         if parsed.errors().is_empty() {
-            let ast = parsed.ast();
+            let ast = parsed.syntax();
 
             let mut visitor = SyntaxLintVisitor {
                 diagnostics,
@@ -86,7 +87,7 @@ pub(crate) fn lint_semantic(db: &dyn LintDb, file_id: FileId) -> QueryResult<Dia
         let context = SemanticLintContext {
             file_id: *file_id,
             source,
-            parsed,
+            parsed: &parsed,
             symbols,
             db,
             diagnostics: RefCell::new(Vec::new()),
@@ -144,9 +145,7 @@ fn lint_bad_overrides(context: &SemanticLintContext) -> QueryResult<()> {
     // TODO we should have a special marker on the real typing module (from typeshed) so if you
     // have your own "typing" module in your project, we don't consider it THE typing module (and
     // same for other stdlib modules that our lint rules care about)
-    let Some(typing_override) =
-        resolve_global_symbol(context.db.upcast(), ModuleName::new("typing"), "override")?
-    else {
+    let Some(typing_override) = context.resolve_global_symbol("typing", "override")? else {
         // TODO once we bundle typeshed, this should be unreachable!()
         return Ok(());
     };
@@ -194,7 +193,7 @@ fn lint_bad_overrides(context: &SemanticLintContext) -> QueryResult<()> {
 pub struct SemanticLintContext<'a> {
     file_id: FileId,
     source: Source,
-    parsed: Parsed,
+    parsed: &'a Parsed<ModModule>,
     symbols: Arc<SymbolTable>,
     db: &'a dyn LintDb,
     diagnostics: RefCell<Vec<String>>,
@@ -209,8 +208,8 @@ impl<'a> SemanticLintContext<'a> {
         self.file_id
     }
 
-    pub fn ast(&self) -> &ModModule {
-        self.parsed.ast()
+    pub fn ast(&self) -> &'a ModModule {
+        self.parsed.syntax()
     }
 
     pub fn symbols(&self) -> &SymbolTable {
@@ -234,6 +233,18 @@ impl<'a> SemanticLintContext<'a> {
     pub fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator<Item = String>) {
         self.diagnostics.get_mut().extend(diagnostics);
     }
+
+    pub fn resolve_global_symbol(
+        &self,
+        module: &str,
+        symbol_name: &str,
+    ) -> QueryResult<Option<GlobalSymbolId>> {
+        let Some(module) = resolve_module(self.db.upcast(), ModuleName::new(module))? else {
+            return Ok(None);
+        };
+
+        resolve_global_symbol(self.db.upcast(), module, symbol_name)
+    }
 }
 
 #[derive(Debug)]

crates/red_knot/src/lint.rs

@@ -1,87 +1,33 @@
 use std::ops::{Deref, DerefMut};
 use std::sync::Arc;
 
-use ruff_python_ast as ast;
-use ruff_python_parser::{Mode, ParseError};
-use ruff_text_size::{Ranged, TextRange};
+use ruff_python_ast::ModModule;
+use ruff_python_parser::Parsed;
 
 use crate::cache::KeyValueCache;
 use crate::db::{QueryResult, SourceDb};
 use crate::files::FileId;
 use crate::source::source_text;
 
-#[derive(Debug, Clone, PartialEq)]
-pub struct Parsed {
-    inner: Arc<ParsedInner>,
-}
-
-#[derive(Debug, PartialEq)]
-struct ParsedInner {
-    ast: ast::ModModule,
-    errors: Vec<ParseError>,
-}
-
-impl Parsed {
-    fn new(ast: ast::ModModule, errors: Vec<ParseError>) -> Self {
-        Self {
-            inner: Arc::new(ParsedInner { ast, errors }),
-        }
-    }
-
-    pub(crate) fn from_text(text: &str) -> Self {
-        let result = ruff_python_parser::parse(text, Mode::Module);
-
-        let (module, errors) = match result {
-            Ok(parsed) => match parsed.into_syntax() {
-                ast::Mod::Module(module) => (module, vec![]),
-                ast::Mod::Expression(expression) => (
-                    ast::ModModule {
-                        range: expression.range(),
-                        body: vec![ast::Stmt::Expr(ast::StmtExpr {
-                            range: expression.range(),
-                            value: expression.body,
-                        })],
-                    },
-                    vec![],
-                ),
-            },
-            Err(errors) => (
-                ast::ModModule {
-                    range: TextRange::default(),
-                    body: Vec::new(),
-                },
-                vec![errors],
-            ),
-        };
-
-        Parsed::new(module, errors)
-    }
-
-    pub fn ast(&self) -> &ast::ModModule {
-        &self.inner.ast
-    }
-
-    pub fn errors(&self) -> &[ParseError] {
-        &self.inner.errors
-    }
-}
-
 #[tracing::instrument(level = "debug", skip(db))]
-pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Parsed> {
+pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Arc<Parsed<ModModule>>> {
     let jar = db.jar()?;
 
     jar.parsed.get(&file_id, |file_id| {
         let source = source_text(db, *file_id)?;
 
-        Ok(Parsed::from_text(source.text()))
+        Ok(Arc::new(ruff_python_parser::parse_unchecked_source(
+            source.text(),
+            source.kind().into(),
+        )))
     })
 }
 
 #[derive(Debug, Default)]
-pub struct ParsedStorage(KeyValueCache<FileId, Parsed>);
+pub struct ParsedStorage(KeyValueCache<FileId, Arc<Parsed<ModModule>>>);
 
 impl Deref for ParsedStorage {
-    type Target = KeyValueCache<FileId, Parsed>;
+    type Target = KeyValueCache<FileId, Arc<Parsed<ModModule>>>;
 
     fn deref(&self) -> &Self::Target {
         &self.0

crates/red_knot/src/parse.rs

@@ -53,6 +53,16 @@ pub enum SourceKind {
     IpyNotebook(Arc<Notebook>),
 }
 
+impl<'a> From<&'a SourceKind> for PySourceType {
+    fn from(value: &'a SourceKind) -> Self {
+        match value {
+            SourceKind::Python(_) => PySourceType::Python,
+            SourceKind::Stub(_) => PySourceType::Stub,
+            SourceKind::IpyNotebook(_) => PySourceType::Ipynb,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub struct Source {
     kind: SourceKind,