Use fancy-regex instead of onig

Author: Keats

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "giallo"
-version = "0.5.2"
+version = "0.6.0"
 edition = "2024"
 license = "EUPL-1.2"
 description = "A code highlighter giving the same output as VSCode"
@@ -19,16 +19,16 @@ include = [
 
 
 [dependencies]
-onig = { package = "onig-regset", version = "6", default-features = false }
 papaya = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
 # Optional dependencies for serialization
 bitcode = { version = "0.6.9", optional = true, features = ["serde"] }
-zstd = { version = "0.13.3", optional = true }
-
+zstd = { version = "0.14", optional = true }
+fancy-regex = "0.19"
 log = { version = "0.4", optional = true }
+regex-syntax = "0.8.11"
 
 [features]
 tools = ["dump"]

benches/highlight_benchmark.rs

@@ -16,7 +16,7 @@ fn highlight_jquery_benchmark(c: &mut Criterion) {
 
     c.bench_function("highlight jquery.js", |b| {
         b.iter(|| {
-            registry.clear_pattern_cache();
+            registry.clear_caches();
             let result = registry.highlight(&jquery_content, &options).unwrap();
             std::hint::black_box(result);
         })
@@ -34,7 +34,7 @@ fn highlight_simple_benchmark(c: &mut Criterion) {
 
     c.bench_function("highlight simple.ts", |b| {
         b.iter(|| {
-            registry.clear_pattern_cache();
+            registry.clear_caches();
             let result = registry.highlight(&ts_content, &options).unwrap();
             std::hint::black_box(result);
         })
@@ -53,7 +53,7 @@ fn highlight_multiple_simple_benchmark(c: &mut Criterion) {
     c.bench_function("highlight multiple simple.ts", |b| {
         b.iter(|| {
             // should not be 5x slower than "highlight simple.ts"
-            registry.clear_pattern_cache();
+            registry.clear_caches();
             let result = registry.highlight(&ts_content, &options).unwrap();
             std::hint::black_box(result);
             let result = registry.highlight(&ts_content, &options).unwrap();
@@ -81,7 +81,7 @@ fn highlight_sample_benchmark(c: &mut Criterion, grammar: &str) {
 
     c.bench_function(&format!("highlight {grammar}.sample"), |b| {
         b.iter(|| {
-            registry.clear_pattern_cache();
+            registry.clear_caches();
             let result = registry.highlight(&content, &options).unwrap();
             std::hint::black_box(result);
         })
@@ -100,13 +100,75 @@ fn highlight_javascript_sample_benchmark(c: &mut Criterion) {
     highlight_sample_benchmark(c, "javascript");
 }
 
+fn highlight_go_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "go");
+}
+
+fn highlight_ruby_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "ruby");
+}
+
+fn highlight_c_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "c");
+}
+
+fn highlight_html_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "html");
+}
+
+fn highlight_css_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "css");
+}
+
+fn highlight_python_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "python");
+}
+
+fn highlight_json_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "json");
+}
+
+fn highlight_tsx_sample_benchmark(c: &mut Criterion) {
+    highlight_sample_benchmark(c, "tsx");
+}
+
+fn highlight_warm_benchmark(c: &mut Criterion) {
+    let registry = Registry::load_from_file("builtin.zst").unwrap();
+    let mut group = c.benchmark_group("highlight warm");
+    for (grammar, path) in [
+        ("typescript", "src/fixtures/samples/simple.ts"),
+        ("javascript", "src/fixtures/samples/jquery.js"),
+        ("rust", "grammars-themes/samples/rust.sample"),
+        ("markdown", "grammars-themes/samples/markdown.sample"),
+        ("python", "grammars-themes/samples/python.sample"),
+        ("tsx", "grammars-themes/samples/tsx.sample"),
+    ] {
+        let content = fs::read_to_string(path).unwrap();
+        let options = HighlightOptions::new(grammar, ThemeVariant::Single("vitesse-black"));
+        registry.highlight(&content, &options).unwrap();
+        group.bench_function(grammar, |b| {
+            b.iter(|| std::hint::black_box(registry.highlight(&content, &options).unwrap()))
+        });
+    }
+    group.finish();
+}
+
 criterion_group!(
     benches,
     highlight_jquery_benchmark,
     highlight_simple_benchmark,
     highlight_multiple_simple_benchmark,
     highlight_rust_sample_benchmark,
     highlight_markdown_sample_benchmark,
-    highlight_javascript_sample_benchmark
+    highlight_javascript_sample_benchmark,
+    highlight_go_sample_benchmark,
+    highlight_ruby_sample_benchmark,
+    highlight_c_sample_benchmark,
+    highlight_html_sample_benchmark,
+    highlight_css_sample_benchmark,
+    highlight_python_sample_benchmark,
+    highlight_json_sample_benchmark,
+    highlight_tsx_sample_benchmark,
+    highlight_warm_benchmark
 );
 criterion_main!(benches);

examples/html_classes.rs

@@ -1,6 +1,7 @@
 use giallo::{HighlightOptions, HtmlRenderer, Registry, RenderOptions, ThemeVariant};
 use std::fs;
 
+#[allow(clippy::single_range_in_vec_init)]
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut registry = Registry::load_from_file("builtin.zst")?;
     registry.link_grammars();

grammars-themes

@@ -1,4 +1,3 @@
-use onig::SearchOptions;
 use std::fmt;
 
 /// We use that as a way to convey both the rule and which anchors should be active
@@ -36,15 +35,21 @@ impl AnchorActive {
         }
     }
 
-    pub fn to_search_options(self) -> SearchOptions {
+    pub fn allow_a(&self) -> bool {
+        matches!(self, AnchorActive::A | AnchorActive::AG)
+    }
+
+    pub fn allow_g(&self) -> bool {
+        matches!(self, AnchorActive::G | AnchorActive::AG)
+    }
+
+    /// The same anchor context with \G deactivated.
+    /// RegexSet handles it on its own (inside fancy-regex) but not the normal Regex
+    /// we walk through
+    pub fn without_g(&self) -> Self {
         match self {
-            AnchorActive::AG => SearchOptions::SEARCH_OPTION_NONE,
-            AnchorActive::A => SearchOptions::SEARCH_OPTION_NOT_BEGIN_POSITION,
-            AnchorActive::G => SearchOptions::SEARCH_OPTION_NOT_BEGIN_STRING,
-            AnchorActive::None => {
-                SearchOptions::SEARCH_OPTION_NOT_BEGIN_STRING
-                    | SearchOptions::SEARCH_OPTION_NOT_BEGIN_POSITION
-            }
+            AnchorActive::A | AnchorActive::AG => AnchorActive::A,
+            AnchorActive::G | AnchorActive::None => AnchorActive::None,
         }
     }
 }

src/tokenizer/anchors.rs src/grammars/anchors.rssrc/tokenizer/anchors.rs renamed to src/grammars/anchors.rs

@@ -0,0 +1,49 @@
+use std::sync::{Arc, OnceLock};
+
+use crate::grammars::engine;
+use crate::grammars::prefilter::{ByteSet, get_byte_set_from_pattern};
+
+/// We want to compile only one of each regex/regexset as they are expensive.
+/// We key by the patterns so they can be reused in multiple grammars/context.
+#[derive(Default, Debug)]
+pub(crate) struct RegexCache {
+    /// We cache the fancy regexes across all grammars
+    regexes: papaya::HashMap<String, OnceLock<Arc<engine::Regex>>>,
+    /// And the sets are cached by the patterns
+    sets: papaya::HashMap<Vec<String>, OnceLock<Arc<engine::RegexSet>>>,
+    /// And the generated byteset for the prefilter
+    first_bytes: papaya::HashMap<String, OnceLock<Option<ByteSet>>>,
+}
+
+impl RegexCache {
+    #[doc(hidden)]
+    pub(crate) fn clear(&self) {
+        self.sets.pin().clear();
+        self.regexes.pin().clear();
+        self.first_bytes.pin().clear();
+    }
+
+    pub fn get_regex(&self, pattern: &str) -> Arc<engine::Regex> {
+        self.regexes
+            .pin()
+            .get_or_insert_with(pattern.to_string(), OnceLock::new)
+            .get_or_init(|| Arc::new(engine::Regex::new(pattern)))
+            .clone()
+    }
+
+    pub fn get_set(&self, patterns: &[String]) -> Arc<engine::RegexSet> {
+        self.sets
+            .pin()
+            .get_or_insert_with(patterns.to_vec(), OnceLock::new)
+            .get_or_init(|| Arc::new(engine::RegexSet::new(patterns)))
+            .clone()
+    }
+
+    pub(crate) fn get_first_bytes(&self, pattern: &str) -> Option<ByteSet> {
+        self.first_bytes
+            .pin()
+            .get_or_insert_with(pattern.to_string(), OnceLock::new)
+            .get_or_init(|| get_byte_set_from_pattern(pattern))
+            .clone()
+    }
+}