Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ prek-consts = { path = "crates/prek-consts", version = "0.3.5" }
prek-identify = { path = "crates/prek-identify", version = "0.3.5" }
prek-pty = { path = "crates/prek-pty", version = "0.3.5" }

aho-corasick = { version = "1.1.4" }
anstream = { version = "0.6.15" }
anstyle-query = { version = "1.1.5" }
anyhow = { version = "1.0.86" }
Expand Down
1 change: 1 addition & 0 deletions crates/prek/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ schemars = ["dep:schemars", "prek-identify/schemars"]
prek-consts = { workspace = true }
prek-identify = { workspace = true, features = ["serde"] }

aho-corasick = { workspace = true }
anstream = { workspace = true }
anstyle-query = { workspace = true }
anyhow = { workspace = true }
Expand Down
51 changes: 47 additions & 4 deletions crates/prek/src/hooks/pre_commit_hooks/detect_private_key.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use std::path::Path;
use std::sync::LazyLock;

use aho_corasick::AhoCorasick;
use anyhow::Result;
use tokio::io::AsyncReadExt;

use crate::hook::Hook;
use crate::hooks::run_concurrent_file_checks;
Expand All @@ -18,6 +21,25 @@ const BLACKLIST: &[&[u8]] = &[
b"BEGIN ENCRYPTED PRIVATE KEY",
b"BEGIN OpenVPN Static key V1",
];
const BUFFER_SIZE: usize = 8192;

// Keep at most the longest marker minus one byte so split matches can span two reads.
const CARRY_CAPACITY: usize = {
let mut max_len = 0;
let mut idx = 0;
while idx < BLACKLIST.len() {
let len = BLACKLIST[idx].len();
if len > max_len {
max_len = len;
}
idx += 1;
}

max_len.saturating_sub(1)
};
static PRIVATE_KEY_MATCHER: LazyLock<AhoCorasick> = LazyLock::new(|| {
AhoCorasick::new(BLACKLIST).expect("private key blacklist patterns should be valid")
});
Comment thread
j178 marked this conversation as resolved.
Resolved
Comment thread
j178 marked this conversation as resolved.
Resolved

pub(crate) async fn detect_private_key(hook: &Hook, filenames: &[&Path]) -> Result<(i32, Vec<u8>)> {
run_concurrent_file_checks(filenames.iter().copied(), *CONCURRENCY, |filename| {
Expand All @@ -26,15 +48,36 @@ pub(crate) async fn detect_private_key(hook: &Hook, filenames: &[&Path]) -> Resu
.await
}

/// Scan the file in chunks while preserving a small tail between reads.
///
/// For example, if one read ends with `BEGIN RSA PRIV` and the next read starts
/// with `ATE KEY`, we keep the tail of the first read, prepend it to the second
/// read, and search the combined window so `BEGIN RSA PRIVATE KEY` is still found.
async fn check_file(file_base: &Path, filename: &Path) -> Result<(i32, Vec<u8>)> {
let content = fs_err::tokio::read(file_base.join(filename)).await?;
let mut file = fs_err::tokio::File::open(file_base.join(filename)).await?;
let mut buf = vec![0u8; BUFFER_SIZE + CARRY_CAPACITY];
let mut carry_len = 0;

// Use memchr's memmem for faster substring search
for pattern in BLACKLIST {
if memchr::memmem::find(&content, pattern).is_some() {
loop {
let bytes_read = file.read(&mut buf[carry_len..]).await?;
if bytes_read == 0 {
break;
}

let search_len = carry_len + bytes_read;
let search_buf = &buf[..search_len];

if PRIVATE_KEY_MATCHER.find(search_buf).is_some() {
let error_message = format!("Private key found: {}\n", filename.display());
return Ok((1, error_message.into_bytes()));
}

// Move the tail of this chunk to the front of the buffer so a key marker
// split across this read and the next read is still seen.
carry_len = CARRY_CAPACITY.min(search_len);
if carry_len > 0 {
buf.copy_within(search_len - carry_len..search_len, 0);
}
Comment thread
j178 marked this conversation as resolved.
Resolved
}

Ok((0, Vec::new()))
Expand Down
16 changes: 7 additions & 9 deletions crates/prek/src/hooks/pre_commit_hooks/fix_end_of_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,21 @@ async fn fix_file(file_base: &Path, filename: &Path) -> Result<(i32, Vec<u8>)> {
.await?;

// If the file is empty, do nothing.
let file_size = file.metadata().await?.len();
let file_size = file.seek(SeekFrom::End(0)).await?;
if file_size == 0 {
return Ok((0, Vec::new()));
}

match find_last_non_ending(&mut file).await? {
match find_last_non_ending(&mut file, file_size).await? {
(None, _) => {
// File contains only line endings, so we can just set it to empty.
file.set_len(0).await?;
file.flush().await?;
file.shutdown().await?;
Ok((1, format!("Fixing {}\n", filename.display()).into_bytes()))
}
(Some(pos), None) => {
// File has some content, but no line ending at the end.
file.seek(SeekFrom::Start(pos + 1)).await?;
file.write_all(b"\n").await?;
file.flush().await?;
file.shutdown().await?;
Ok((1, format!("Fixing {}\n", filename.display()).into_bytes()))
}
(Some(pos), Some(line_ending)) => {
Expand Down Expand Up @@ -71,20 +67,22 @@ fn determine_line_ending(first: u8, second: u8) -> Option<&'static str> {

/// Searches for the last non-line-ending character in the file.
/// Returns the position of the last non-line-ending character and the line ending type.
async fn find_last_non_ending<T>(reader: &mut T) -> Result<(Option<u64>, Option<&str>)>
async fn find_last_non_ending<T>(
reader: &mut T,
data_len: u64,
) -> Result<(Option<u64>, Option<&str>)>
where
T: AsyncRead + AsyncSeek + Unpin,
{
const MAX_SCAN_SIZE: usize = 4 * 1024; // 4KB

let data_len = reader.seek(SeekFrom::End(0)).await?;
if data_len == 0 {
return Ok((None, None));
}

let mut read_len = 0;
let mut next_char = 0;
let mut buf = vec![0u8; MAX_SCAN_SIZE];
let mut buf = [0u8; MAX_SCAN_SIZE];
let mut line_ending = None;

while read_len < data_len {
Expand Down
1 change: 1 addition & 0 deletions scripts/hyperfine-run-benchmarks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ INDIVIDUAL_HOOKS=(
"check-yaml"
"check-toml"
"check-xml"
"detect-private-key"
"fix-byte-order-marker"
)

Expand Down
Loading