From 4ea26ba73b0c51252242078c13449d3e8e59ede9 Mon Sep 17 00:00:00 2001 From: Jo <10510431+j178@users.noreply.github.com> Date: Sun, 15 Mar 2026 15:29:50 +0800 Subject: [PATCH 1/2] Optimize `detect_private_key` by chunked reading and using aho-corasick Benchmark `detect-private-key` in hyperfine script --- Cargo.lock | 1 + Cargo.toml | 1 + crates/prek/Cargo.toml | 1 + .../pre_commit_hooks/detect_private_key.rs | 51 +++++++++++++++++-- scripts/hyperfine-run-benchmarks.sh | 1 + 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1016b9bf3..33e642cd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2113,6 +2113,7 @@ dependencies = [ name = "prek" version = "0.3.5" dependencies = [ + "aho-corasick", "anstream", "anstyle-query", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index d875260e6..f270a0c31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ prek-consts = { path = "crates/prek-consts", version = "0.3.5" } prek-identify = { path = "crates/prek-identify", version = "0.3.5" } prek-pty = { path = "crates/prek-pty", version = "0.3.5" } +aho-corasick = { version = "1.1.4" } anstream = { version = "0.6.15" } anstyle-query = { version = "1.1.5" } anyhow = { version = "1.0.86" } diff --git a/crates/prek/Cargo.toml b/crates/prek/Cargo.toml index d0fe86acf..59a88e504 100644 --- a/crates/prek/Cargo.toml +++ b/crates/prek/Cargo.toml @@ -26,6 +26,7 @@ schemars = ["dep:schemars", "prek-identify/schemars"] prek-consts = { workspace = true } prek-identify = { workspace = true, features = ["serde"] } +aho-corasick = { workspace = true } anstream = { workspace = true } anstyle-query = { workspace = true } anyhow = { workspace = true } diff --git a/crates/prek/src/hooks/pre_commit_hooks/detect_private_key.rs b/crates/prek/src/hooks/pre_commit_hooks/detect_private_key.rs index ac9a8be71..2ec749399 100644 --- a/crates/prek/src/hooks/pre_commit_hooks/detect_private_key.rs +++ b/crates/prek/src/hooks/pre_commit_hooks/detect_private_key.rs @@ -1,6 +1,9 @@ use std::path::Path; +use std::sync::LazyLock; +use aho_corasick::AhoCorasick; use anyhow::Result; +use tokio::io::AsyncReadExt; use crate::hook::Hook; use crate::hooks::run_concurrent_file_checks; @@ -18,6 +21,25 @@ const BLACKLIST: &[&[u8]] = &[ b"BEGIN ENCRYPTED PRIVATE KEY", b"BEGIN OpenVPN Static key V1", ]; +const BUFFER_SIZE: usize = 8192; + +// Keep at most the longest marker minus one byte so split matches can span two reads. +const CARRY_CAPACITY: usize = { + let mut max_len = 0; + let mut idx = 0; + while idx < BLACKLIST.len() { + let len = BLACKLIST[idx].len(); + if len > max_len { + max_len = len; + } + idx += 1; + } + + max_len.saturating_sub(1) +}; +static PRIVATE_KEY_MATCHER: LazyLock = LazyLock::new(|| { + AhoCorasick::new(BLACKLIST).expect("private key blacklist patterns should be valid") +}); pub(crate) async fn detect_private_key(hook: &Hook, filenames: &[&Path]) -> Result<(i32, Vec)> { run_concurrent_file_checks(filenames.iter().copied(), *CONCURRENCY, |filename| { @@ -26,15 +48,36 @@ pub(crate) async fn detect_private_key(hook: &Hook, filenames: &[&Path]) -> Resu .await } +/// Scan the file in chunks while preserving a small tail between reads. +/// +/// For example, if one read ends with `BEGIN RSA PRIV` and the next read starts +/// with `ATE KEY`, we keep the tail of the first read, prepend it to the second +/// read, and search the combined window so `BEGIN RSA PRIVATE KEY` is still found. async fn check_file(file_base: &Path, filename: &Path) -> Result<(i32, Vec)> { - let content = fs_err::tokio::read(file_base.join(filename)).await?; + let mut file = fs_err::tokio::File::open(file_base.join(filename)).await?; + let mut buf = vec![0u8; BUFFER_SIZE + CARRY_CAPACITY]; + let mut carry_len = 0; - // Use memchr's memmem for faster substring search - for pattern in BLACKLIST { - if memchr::memmem::find(&content, pattern).is_some() { + loop { + let bytes_read = file.read(&mut buf[carry_len..]).await?; + if bytes_read == 0 { + break; + } + + let search_len = carry_len + bytes_read; + let search_buf = &buf[..search_len]; + + if PRIVATE_KEY_MATCHER.find(search_buf).is_some() { let error_message = format!("Private key found: {}\n", filename.display()); return Ok((1, error_message.into_bytes())); } + + // Move the tail of this chunk to the front of the buffer so a key marker + // split across this read and the next read is still seen. + carry_len = CARRY_CAPACITY.min(search_len); + if carry_len > 0 { + buf.copy_within(search_len - carry_len..search_len, 0); + } } Ok((0, Vec::new())) diff --git a/scripts/hyperfine-run-benchmarks.sh b/scripts/hyperfine-run-benchmarks.sh index 311cf9704..7a30cd94e 100755 --- a/scripts/hyperfine-run-benchmarks.sh +++ b/scripts/hyperfine-run-benchmarks.sh @@ -213,6 +213,7 @@ INDIVIDUAL_HOOKS=( "check-yaml" "check-toml" "check-xml" + "detect-private-key" "fix-byte-order-marker" ) From f4419835312408af2d0fa17cc5cc24cb9c8cc80d Mon Sep 17 00:00:00 2001 From: Jo <10510431+j178@users.noreply.github.com> Date: Sun, 15 Mar 2026 17:22:24 +0800 Subject: [PATCH 2/2] Optimize `fix_end_of_file` by reducing file operations (#1792) Reduce file operations in fix_end_of_file. --- .../hooks/pre_commit_hooks/fix_end_of_file.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/prek/src/hooks/pre_commit_hooks/fix_end_of_file.rs b/crates/prek/src/hooks/pre_commit_hooks/fix_end_of_file.rs index 9af40e65e..a8675519c 100644 --- a/crates/prek/src/hooks/pre_commit_hooks/fix_end_of_file.rs +++ b/crates/prek/src/hooks/pre_commit_hooks/fix_end_of_file.rs @@ -23,25 +23,21 @@ async fn fix_file(file_base: &Path, filename: &Path) -> Result<(i32, Vec)> { .await?; // If the file is empty, do nothing. - let file_size = file.metadata().await?.len(); + let file_size = file.seek(SeekFrom::End(0)).await?; if file_size == 0 { return Ok((0, Vec::new())); } - match find_last_non_ending(&mut file).await? { + match find_last_non_ending(&mut file, file_size).await? { (None, _) => { // File contains only line endings, so we can just set it to empty. file.set_len(0).await?; - file.flush().await?; - file.shutdown().await?; Ok((1, format!("Fixing {}\n", filename.display()).into_bytes())) } (Some(pos), None) => { // File has some content, but no line ending at the end. file.seek(SeekFrom::Start(pos + 1)).await?; file.write_all(b"\n").await?; - file.flush().await?; - file.shutdown().await?; Ok((1, format!("Fixing {}\n", filename.display()).into_bytes())) } (Some(pos), Some(line_ending)) => { @@ -71,20 +67,22 @@ fn determine_line_ending(first: u8, second: u8) -> Option<&'static str> { /// Searches for the last non-line-ending character in the file. /// Returns the position of the last non-line-ending character and the line ending type. -async fn find_last_non_ending(reader: &mut T) -> Result<(Option, Option<&str>)> +async fn find_last_non_ending( + reader: &mut T, + data_len: u64, +) -> Result<(Option, Option<&str>)> where T: AsyncRead + AsyncSeek + Unpin, { const MAX_SCAN_SIZE: usize = 4 * 1024; // 4KB - let data_len = reader.seek(SeekFrom::End(0)).await?; if data_len == 0 { return Ok((None, None)); } let mut read_len = 0; let mut next_char = 0; - let mut buf = vec![0u8; MAX_SCAN_SIZE]; + let mut buf = [0u8; MAX_SCAN_SIZE]; let mut line_ending = None; while read_len < data_len {