Use fancy-regex instead of onig

Use changes from fancy-regex lazy branch

Prebuild bytesets and update benches

Do not build a regexset for a single regex

Author: Keats

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "giallo"
-version = "0.5.2"
+version = "0.6.0"
 edition = "2024"
 license = "EUPL-1.2"
 description = "A code highlighter giving the same output as VSCode"
@@ -19,17 +19,19 @@ include = [
 
 
 [dependencies]
-onig = { package = "onig-regset", version = "6", default-features = false }
 papaya = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
 # Optional dependencies for serialization
 bitcode = { version = "0.6.9", optional = true, features = ["serde"] }
-zstd = { version = "0.13.3", optional = true }
-
+zstd = { version = "0.14", optional = true }
+fancy-regex = "0.19"
 log = { version = "0.4", optional = true }
 
+[patch.crates-io]
+fancy-regex = { git = "https://github.com/fancy-regex/fancy-regex" }
+
 [features]
 tools = ["dump"]
 debug = ["dep:log"]

benches/highlight_benchmark.rs

@@ -1,26 +1,35 @@
-use criterion::{Criterion, criterion_group, criterion_main};
-use giallo::{HighlightOptions, Registry, ThemeVariant};
 use std::fs;
 
-fn highlight_jquery_benchmark(c: &mut Criterion) {
-    // Load registry once for all benchmarks
-    let mut registry =
-        Registry::load_from_file("builtin.zst").expect("Failed to load registry from builtin.zst");
-    registry.link_grammars();
-
-    // Read jQuery file content once
-    let jquery_content =
-        fs::read_to_string("src/fixtures/samples/jquery.js").expect("Failed to read jQuery file");
-
-    let options = HighlightOptions::new("javascript", ThemeVariant::Single("vitesse-black"));
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use giallo::{HighlightOptions, Registry, ThemeVariant};
 
-    c.bench_function("highlight jquery.js", |b| {
-        b.iter(|| {
-            registry.clear_pattern_cache();
-            let result = registry.highlight(&jquery_content, &options).unwrap();
-            std::hint::black_box(result);
-        })
-    });
+const SAMPLES: &[(&str, &str)] = &[
+    ("javascript", "src/fixtures/samples/jquery.js"),
+    ("javascript", "grammars-themes/samples/javascript.sample"),
+    ("typescript", "grammars-themes/samples/typescript.sample"),
+    ("tsx", "grammars-themes/samples/tsx.sample"),
+    ("rust", "grammars-themes/samples/rust.sample"),
+    ("c", "grammars-themes/samples/c.sample"),
+    ("markdown", "grammars-themes/samples/markdown.sample"),
+    ("python", "grammars-themes/samples/python.sample"),
+    ("html", "grammars-themes/samples/html.sample"),
+    ("css", "grammars-themes/samples/css.sample"),
+    ("ruby", "grammars-themes/samples/ruby.sample"),
+    ("go", "grammars-themes/samples/go.sample"),
+    ("astro", "grammars-themes/samples/astro.sample"),
+    ("c#", "grammars-themes/samples/csharp.sample"),
+    ("java", "grammars-themes/samples/java.sample"),
+    ("php", "grammars-themes/samples/php.sample"),
+    ("json", "grammars-themes/samples/json.sample"),
+    ("shellscript", "grammars-themes/samples/shellscript.sample"),
+];
+
+fn bench_name(grammar: &'static str, path: &str) -> &'static str {
+    if path.contains("jquery") {
+        "jquery"
+    } else {
+        grammar
+    }
 }
 
 fn highlight_simple_benchmark(c: &mut Criterion) {
@@ -33,11 +42,11 @@ fn highlight_simple_benchmark(c: &mut Criterion) {
     let options = HighlightOptions::new("typescript", ThemeVariant::Single("vitesse-black"));
 
     c.bench_function("highlight simple.ts", |b| {
-        b.iter(|| {
-            registry.clear_pattern_cache();
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-        })
+        b.iter_batched(
+            || registry.clear_caches(),
+            |()| std::hint::black_box(registry.highlight(&ts_content, &options).unwrap()),
+            BatchSize::PerIteration,
+        )
     });
 }
 
@@ -51,62 +60,57 @@ fn highlight_multiple_simple_benchmark(c: &mut Criterion) {
     let options = HighlightOptions::new("typescript", ThemeVariant::Single("vitesse-black"));
 
     c.bench_function("highlight multiple simple.ts", |b| {
-        b.iter(|| {
-            // should not be 5x slower than "highlight simple.ts"
-            registry.clear_pattern_cache();
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-            let result = registry.highlight(&ts_content, &options).unwrap();
-            std::hint::black_box(result);
-        })
+        b.iter_batched(
+            || registry.clear_caches(),
+            |()| {
+                // should not be 5x slower than "highlight simple.ts"
+                for _ in 0..5 {
+                    std::hint::black_box(registry.highlight(&ts_content, &options).unwrap());
+                }
+            },
+            BatchSize::PerIteration,
+        )
     });
 }
 
-fn highlight_sample_benchmark(c: &mut Criterion, grammar: &str) {
-    let mut registry =
-        Registry::load_from_file("builtin.zst").expect("Failed to load registry from builtin.zst");
-    registry.link_grammars();
-
-    let sample_path = format!("grammars-themes/samples/{grammar}.sample");
-    let content = fs::read_to_string(&sample_path)
-        .unwrap_or_else(|_| panic!("Failed to read sample {sample_path}"));
-
-    let options = HighlightOptions::new(grammar, ThemeVariant::Single("vitesse-black"));
-
-    c.bench_function(&format!("highlight {grammar}.sample"), |b| {
-        b.iter(|| {
-            registry.clear_pattern_cache();
-            let result = registry.highlight(&content, &options).unwrap();
-            std::hint::black_box(result);
-        })
-    });
-}
-
-fn highlight_rust_sample_benchmark(c: &mut Criterion) {
-    highlight_sample_benchmark(c, "rust");
-}
-
-fn highlight_markdown_sample_benchmark(c: &mut Criterion) {
-    highlight_sample_benchmark(c, "markdown");
+fn highlight_cold_benchmark(c: &mut Criterion) {
+    let registry = Registry::load_from_file("builtin.zst").unwrap();
+    let mut group = c.benchmark_group("highlight cold");
+    group.sample_size(50);
+    for &(grammar, path) in SAMPLES {
+        let content = fs::read_to_string(path).unwrap();
+        let options = HighlightOptions::new(grammar, ThemeVariant::Single("vitesse-black"));
+        registry.highlight(&content, &options).unwrap();
+        group.bench_function(bench_name(grammar, path), |b| {
+            b.iter_batched(
+                || registry.clear_caches(),
+                |()| std::hint::black_box(registry.highlight(&content, &options).unwrap()),
+                BatchSize::PerIteration,
+            )
+        });
+    }
+    group.finish();
 }
 
-fn highlight_javascript_sample_benchmark(c: &mut Criterion) {
-    highlight_sample_benchmark(c, "javascript");
+fn highlight_warm_benchmark(c: &mut Criterion) {
+    let registry = Registry::load_from_file("builtin.zst").unwrap();
+    let mut group = c.benchmark_group("highlight warm");
+    for &(grammar, path) in SAMPLES {
+        let content = fs::read_to_string(path).unwrap();
+        let options = HighlightOptions::new(grammar, ThemeVariant::Single("vitesse-black"));
+        registry.highlight(&content, &options).unwrap();
+        group.bench_function(bench_name(grammar, path), |b| {
+            b.iter(|| std::hint::black_box(registry.highlight(&content, &options).unwrap()))
+        });
+    }
+    group.finish();
 }
 
 criterion_group!(
     benches,
-    highlight_jquery_benchmark,
     highlight_simple_benchmark,
     highlight_multiple_simple_benchmark,
-    highlight_rust_sample_benchmark,
-    highlight_markdown_sample_benchmark,
-    highlight_javascript_sample_benchmark
+    highlight_cold_benchmark,
+    highlight_warm_benchmark,
 );
 criterion_main!(benches);

examples/html_classes.rs

@@ -1,6 +1,7 @@
 use giallo::{HighlightOptions, HtmlRenderer, Registry, RenderOptions, ThemeVariant};
 use std::fs;
 
+#[allow(clippy::single_range_in_vec_init)]
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut registry = Registry::load_from_file("builtin.zst")?;
     registry.link_grammars();

grammars-themes

@@ -1,4 +1,3 @@
-use onig::SearchOptions;
 use std::fmt;
 
 /// We use that as a way to convey both the rule and which anchors should be active
@@ -36,15 +35,21 @@ impl AnchorActive {
         }
     }
 
-    pub fn to_search_options(self) -> SearchOptions {
+    pub fn allow_a(&self) -> bool {
+        matches!(self, AnchorActive::A | AnchorActive::AG)
+    }
+
+    pub fn allow_g(&self) -> bool {
+        matches!(self, AnchorActive::G | AnchorActive::AG)
+    }
+
+    /// The same anchor context with \G deactivated.
+    /// RegexSet handles it on its own (inside fancy-regex) but not the normal Regex
+    /// we walk through
+    pub fn without_g(&self) -> Self {
         match self {
-            AnchorActive::AG => SearchOptions::SEARCH_OPTION_NONE,
-            AnchorActive::A => SearchOptions::SEARCH_OPTION_NOT_BEGIN_POSITION,
-            AnchorActive::G => SearchOptions::SEARCH_OPTION_NOT_BEGIN_STRING,
-            AnchorActive::None => {
-                SearchOptions::SEARCH_OPTION_NOT_BEGIN_STRING
-                    | SearchOptions::SEARCH_OPTION_NOT_BEGIN_POSITION
-            }
+            AnchorActive::A | AnchorActive::AG => AnchorActive::A,
+            AnchorActive::G | AnchorActive::None => AnchorActive::None,
         }
     }
 }

src/tokenizer/anchors.rs src/grammars/anchors.rssrc/tokenizer/anchors.rs renamed to src/grammars/anchors.rs

@@ -0,0 +1,40 @@
+use std::sync::{Arc, OnceLock};
+
+use crate::grammars::engine;
+
+/// We want to compile only one of each regex/regexset as they are expensive.
+/// We key by the patterns so they can be reused in multiple grammars/context.
+#[derive(Default, Debug)]
+pub(crate) struct RegexCache {
+    /// We cache the fancy regexes across all grammars
+    regexes: papaya::HashMap<String, OnceLock<Arc<engine::Regex>>>,
+    /// And the sets are cached by the patterns
+    sets: papaya::HashMap<Vec<String>, OnceLock<Arc<engine::RegexSet>>>,
+}
+
+impl RegexCache {
+    #[doc(hidden)]
+    pub(crate) fn clear(&self) {
+        self.sets.pin().clear();
+        self.regexes.pin().clear();
+    }
+
+    pub fn get_regex(&self, pattern: &str) -> Arc<engine::Regex> {
+        self.regexes
+            .pin()
+            .get_or_insert_with(pattern.to_string(), OnceLock::new)
+            .get_or_init(|| Arc::new(engine::Regex::new(pattern)))
+            .clone()
+    }
+
+    pub fn get_set(&self, patterns: &[String]) -> Arc<engine::RegexSet> {
+        self.sets
+            .pin()
+            .get_or_insert_with(patterns.to_vec(), OnceLock::new)
+            .get_or_init(|| {
+                let regexes: Vec<_> = patterns.iter().map(|p| self.get_regex(p)).collect();
+                Arc::new(engine::RegexSet::from_regexes(&regexes))
+            })
+            .clone()
+    }
+}