Skip to content

Commit dfdb09c

Browse files
KeatsKeats
committed
wip
1 parent 415eaab commit dfdb09c

9 files changed

Lines changed: 824 additions & 46 deletions

File tree

‎src/lazy/byte_set.rs‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
use regex_syntax::hir::{Class, ClassUnicode, ClassUnicodeRange, Hir, HirKind};
2+
3+
use crate::to_hir::{expr_to_hir, HirCtx};
4+
use crate::{Expr, LookAround, RegexOptions};
5+
6+
/// A set of bytes, stored as a 256 bits bitmap
7+
#[derive(Clone, Debug, PartialEq, Eq, Default)]
8+
pub(crate) struct ByteSet([u64; 4]);
9+
10+
impl ByteSet {
11+
pub(crate) fn new() -> Self {
12+
Self([0; 4])
13+
}
14+
15+
pub(crate) fn insert(&mut self, byte: u8) {
16+
self.0[(byte >> 6) as usize] |= 1u64 << (byte & 63);
17+
}
18+
19+
pub(crate) fn is_empty(&self) -> bool {
20+
self.0 == [0; 4]
21+
}
22+
23+
pub(crate) fn contains(&self, byte: u8) -> bool {
24+
self.0[(byte >> 6) as usize] & (1u64 << (byte & 63)) != 0
25+
}
26+
27+
pub(crate) fn union(&mut self, other: &Self) {
28+
for (a, b) in self.0.iter_mut().zip(&other.0) {
29+
*a |= *b;
30+
}
31+
}
32+
33+
/// Iterates over the bytes in the set, in ascending order
34+
pub(crate) fn iter(&self) -> impl Iterator<Item = u8> + '_ {
35+
set_bits(&self.0).map(|i| i as u8)
36+
}
37+
}
38+
39+
/// Get an iterator for the bits set, starting from the lowest
40+
fn bits_of(mut bits: u64) -> impl Iterator<Item = usize> {
41+
core::iter::from_fn(move || {
42+
(bits != 0).then(|| {
43+
let bit = bits.trailing_zeros() as usize;
44+
// clear the lowest set bit so the next call gets the following one
45+
bits &= bits - 1;
46+
bit
47+
})
48+
})
49+
}
50+
51+
/// Get an iterator for the indices of the bits set, starting from the lowest
52+
pub(crate) fn set_bits(words: &[u64]) -> impl Iterator<Item = usize> + '_ {
53+
words
54+
.iter()
55+
.enumerate()
56+
.flat_map(|(i, &word)| bits_of(word).map(move |bit| i * 64 + bit))
57+
}
58+
59+
#[derive(Debug, Clone, PartialEq, Eq)]
60+
enum Start {
61+
/// Every match must start with a byte from the set
62+
Definite(ByteSet),
63+
/// The match can be empty but otherwise it starts with a byte from the set
64+
MaybeEmpty(ByteSet),
65+
/// Could be anything.
66+
Bail,
67+
}
68+
69+
/// Same logic as fancy_start but on regex-syntax HIR instead.
70+
fn regex_syntax_start(expr: &Hir) -> Start {
71+
match expr.kind() {
72+
// 0 len match
73+
HirKind::Empty | HirKind::Look(_) => Start::MaybeEmpty(ByteSet::new()),
74+
HirKind::Literal(l) => {
75+
if let Some(first) = l.0.first() {
76+
let mut set = ByteSet::new();
77+
set.insert(*first);
78+
Start::Definite(set)
79+
} else {
80+
Start::MaybeEmpty(ByteSet::new())
81+
}
82+
}
83+
HirKind::Class(class) => {
84+
let mut set = ByteSet::new();
85+
match class {
86+
Class::Unicode(cls) => {
87+
for r in cls.ranges() {
88+
if r.start().is_ascii() {
89+
for i in (r.start() as u32)..=(r.end() as u32).min(0x7F) {
90+
set.insert(i as u8);
91+
}
92+
}
93+
94+
if !r.end().is_ascii() {
95+
// unicode, just insert lead bytes range
96+
for i in 0xC2..=0xF4 {
97+
set.insert(i as u8);
98+
}
99+
}
100+
}
101+
}
102+
Class::Bytes(cls) => {
103+
for r in cls.ranges() {
104+
for i in r.start()..=r.end() {
105+
set.insert(i);
106+
}
107+
}
108+
}
109+
}
110+
111+
if set.is_empty() {
112+
Start::Bail
113+
} else {
114+
Start::Definite(set)
115+
}
116+
}
117+
HirKind::Capture(c) => regex_syntax_start(&c.sub),
118+
HirKind::Repetition(r) => {
119+
let child_start = regex_syntax_start(&r.sub);
120+
if r.min == 0 {
121+
// if we allow 0 reps then it's not definite
122+
match child_start {
123+
Start::Definite(set) | Start::MaybeEmpty(set) => Start::MaybeEmpty(set),
124+
Start::Bail => Start::Bail,
125+
}
126+
} else {
127+
child_start
128+
}
129+
}
130+
HirKind::Concat(c) => {
131+
let mut set = ByteSet::new();
132+
for expr in c {
133+
match regex_syntax_start(expr) {
134+
Start::Definite(mut s) => {
135+
// We only care about the first byte so as soon as we hit a definite match
136+
// we don't need to continue processing the rest
137+
s.union(&set);
138+
return Start::Definite(s);
139+
}
140+
Start::MaybeEmpty(s) => {
141+
set.union(&s);
142+
}
143+
Start::Bail => {
144+
// If we even get one bail, we bail everything
145+
return Start::Bail;
146+
}
147+
}
148+
}
149+
Start::MaybeEmpty(set)
150+
}
151+
HirKind::Alternation(exprs) => {
152+
let mut total = None;
153+
for expr in exprs {
154+
let expr_start = regex_syntax_start(expr);
155+
total = Some(match (total, expr_start) {
156+
(None, b) => b,
157+
(Some(Start::Bail), _) | (Some(_), Start::Bail) => Start::Bail,
158+
(Some(Start::Definite(mut a)), Start::Definite(b)) => {
159+
a.union(&b);
160+
Start::Definite(a)
161+
}
162+
(Some(Start::Definite(mut a)), Start::MaybeEmpty(b))
163+
| (Some(Start::MaybeEmpty(mut a)), Start::Definite(b))
164+
| (Some(Start::MaybeEmpty(mut a)), Start::MaybeEmpty(b)) => {
165+
a.union(&b);
166+
Start::MaybeEmpty(a)
167+
}
168+
});
169+
}
170+
total.unwrap_or(Start::MaybeEmpty(ByteSet::new()))
171+
}
172+
}
173+
}
174+
175+
#[inline]
176+
fn first_utf8_byte(c: char) -> u8 {
177+
let mut buf = [0u8; 4];
178+
c.encode_utf8(&mut buf).as_bytes()[0]
179+
}
180+
181+
/// TODO: what to do with low selective things like `\S+` or anything negated like `[^a]` that can match pretty much any chars?
182+
fn fancy_start(expr: &Expr, ctx: &mut HirCtx) -> Start {
183+
match expr {
184+
Expr::Empty | Expr::Assertion(_) | Expr::KeepOut | Expr::ContinueFromPreviousMatchEnd => {
185+
Start::MaybeEmpty(ByteSet::new())
186+
}
187+
Expr::Literal { val, casei } => {
188+
let Some(first) = val.chars().next() else {
189+
return Start::MaybeEmpty(ByteSet::new());
190+
};
191+
let mut set = ByteSet::new();
192+
if *casei {
193+
let mut class = ClassUnicode::new([ClassUnicodeRange::new(first, first)]);
194+
if class.try_case_fold_simple().is_err() {
195+
return Start::Bail;
196+
}
197+
for range in class.ranges() {
198+
for c in range.start()..=range.end() {
199+
set.insert(first_utf8_byte(c));
200+
}
201+
}
202+
} else {
203+
set.insert(first_utf8_byte(first));
204+
}
205+
Start::Definite(set)
206+
}
207+
Expr::Delegate { .. } => match expr_to_hir(expr, ctx) {
208+
Some(hir) => regex_syntax_start(&hir),
209+
None => Start::Bail,
210+
},
211+
212+
Expr::Concat(exprs) => {
213+
let mut set = ByteSet::new();
214+
for expr in exprs {
215+
match fancy_start(expr, ctx) {
216+
Start::Definite(mut s) => {
217+
// We only care about the first byte so as soon as we hit a definite match
218+
// we don't need to continue processing the rest
219+
s.union(&set);
220+
return Start::Definite(s);
221+
}
222+
Start::MaybeEmpty(s) => {
223+
set.union(&s);
224+
}
225+
Start::Bail => {
226+
// If we even get one bail, we bail everything
227+
return Start::Bail;
228+
}
229+
}
230+
}
231+
Start::MaybeEmpty(set)
232+
}
233+
234+
Expr::Alt(exprs) => {
235+
let mut total = None;
236+
for expr in exprs {
237+
let expr_start = fancy_start(expr, ctx);
238+
total = Some(match (total, expr_start) {
239+
(None, b) => b,
240+
(Some(Start::Bail), _) | (Some(_), Start::Bail) => Start::Bail,
241+
(Some(Start::Definite(mut a)), Start::Definite(b)) => {
242+
a.union(&b);
243+
Start::Definite(a)
244+
}
245+
(Some(Start::Definite(mut a)), Start::MaybeEmpty(b))
246+
| (Some(Start::MaybeEmpty(mut a)), Start::Definite(b))
247+
| (Some(Start::MaybeEmpty(mut a)), Start::MaybeEmpty(b)) => {
248+
a.union(&b);
249+
Start::MaybeEmpty(a)
250+
}
251+
});
252+
}
253+
total.unwrap_or(Start::MaybeEmpty(ByteSet::new()))
254+
}
255+
256+
Expr::Group(child) => fancy_start(child, ctx),
257+
Expr::AtomicGroup(child) => fancy_start(child, ctx),
258+
259+
Expr::Repeat { child, lo, .. } => {
260+
let child_start = fancy_start(child, ctx);
261+
if *lo == 0 {
262+
// if we allow 0 reps then it's not definite
263+
match child_start {
264+
Start::Definite(set) | Start::MaybeEmpty(set) => Start::MaybeEmpty(set),
265+
Start::Bail => Start::Bail,
266+
}
267+
} else {
268+
child_start
269+
}
270+
}
271+
Expr::LookAround(expr, lookaround) => match lookaround {
272+
LookAround::LookAhead => match fancy_start(expr, ctx) {
273+
Start::Definite(set) => Start::Definite(set),
274+
_ => Start::MaybeEmpty(ByteSet::new()),
275+
},
276+
_ => Start::MaybeEmpty(ByteSet::new()),
277+
},
278+
_ => Start::Bail,
279+
}
280+
}
281+
282+
pub(crate) fn byte_set_from_expr(expr: &Expr, options: &RegexOptions) -> Option<ByteSet> {
283+
let mut hir_ctx = HirCtx::from(options);
284+
match fancy_start(expr, &mut hir_ctx) {
285+
Start::Definite(set) if !set.is_empty() => Some(set),
286+
_ => None,
287+
}
288+
}

0 commit comments

Comments
 (0)