Add an enum for mode strategy

Author: Keats

src/grammars/matcher.rs

@@ -1,10 +1,30 @@
-use std::sync::{Arc, OnceLock};
-
 use crate::grammars::anchors::AnchorActive;
 use crate::grammars::caches::RegexCache;
 use crate::grammars::engine::CaptureSpans;
 use crate::grammars::prefilter::Prefilter;
 use crate::grammars::{GlobalRuleRef, engine};
+use std::sync::{Arc, OnceLock};
+
+/// How will giallo find the next matching pattern
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Default)]
+pub enum MatchStrategy {
+    /// giallo will check every pattern individually when it can and fallback to a RegexSet when it cannot
+    ///
+    /// This is fast when giallo is cold and uses less memory.
+    /// This should be used by CLIs highlighting a few things and exiting or if you are memory constrained.
+    #[default]
+    Walk,
+    /// giallo will only use a RegexSet
+    ///
+    /// Compared to [MatchStrategy::Walk]:
+    /// 1. Initial RegexSet compilation is _very_ slow: first highlight will likely be around 10x slower
+    /// 2. It will use *much* more memory  (2-10x more) growing with the number of languages that have highlighted
+    /// 3. Warm highlights will be between 15-60% faster, dependent on language
+    ///
+    /// This should only be used if you're using giallo in a long-lived process and you're okay with
+    /// potentially 10GB+ of ram used just for giallo with a few dozen languages.
+    Set,
+}
 
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct RuleMatch {
@@ -132,20 +152,27 @@ pub struct RuleMatcher {
 }
 
 impl RuleMatcher {
-    pub fn new(items: Vec<(GlobalRuleRef, String)>, cache: Arc<RegexCache>) -> Self {
+    pub fn new(
+        items: Vec<(GlobalRuleRef, String)>,
+        cache: Arc<RegexCache>,
+        strategy: MatchStrategy,
+    ) -> Self {
         if items.is_empty() {
             return RuleMatcher::default();
         }
 
         let (rule_refs, patterns): (Vec<_>, Vec<_>) = items.into_iter().unzip();
-        let sets: Vec<_> = patterns.iter().map(|p| cache.get_first_bytes(p)).collect();
 
-        let (prefilter, remaining) =
+        let (prefilter, remaining) = if strategy == MatchStrategy::Walk {
+            let sets: Vec<_> = patterns.iter().map(|p| cache.get_first_bytes(p)).collect();
             if let Some((prefilter, remaining)) = Prefilter::from_byte_sets(&sets) {
                 (prefilter, remaining)
             } else {
                 (Prefilter::default(), (0..patterns.len()).collect())
-            };
+            }
+        } else {
+            (Prefilter::default(), (0..patterns.len()).collect())
+        };
 
         let has_walk = remaining.len() < patterns.len();
         let remainder = (!remaining.is_empty()).then(|| {

src/grammars/mod.rs

@@ -10,6 +10,6 @@ mod raw;
 
 pub use compiled::*;
 pub use injections::InjectionPrecedence;
-pub use matcher::{RuleMatch, RuleMatcher};
+pub use matcher::{MatchStrategy, RuleMatch, RuleMatcher};
 pub use pattern::resolve_backreferences;
 pub use raw::RawGrammar;

src/lib.rs

@@ -34,6 +34,7 @@ mod renderers;
 mod tokenizer;
 
 pub use error::Error;
+pub use grammars::MatchStrategy;
 pub use highlight::HighlightedText;
 pub use markdown_fence::{ParsedFence, parse_markdown_fence};
 pub use registry::{HighlightOptions, HighlightedCode, PLAIN_GRAMMAR_NAME, Registry};

src/registry.rs

@@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet};
 use std::path::Path;
 use std::sync::Arc;
 
+use crate::MatchStrategy;
 use crate::error::{Error, GialloResult};
 use crate::grammars::caches::RegexCache;
 use crate::grammars::{
@@ -10,14 +11,13 @@ use crate::grammars::{
     resolve_external_references,
 };
 use crate::highlight::{HighlightedText, Highlighter, MergingOptions};
-use serde::{Deserialize, Serialize};
-
 #[cfg(feature = "dump")]
 use crate::scope::ScopeRepository;
 use crate::scope::{Scope, ScopeInterner};
 use crate::themes::css::{DARK_SUFFIX, LIGHT_SUFFIX};
 use crate::themes::{CompiledTheme, RawTheme, ThemeVariant};
 use crate::tokenizer::{Token, Tokenizer};
+use serde::{Deserialize, Serialize};
 
 #[cfg(feature = "dump")]
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -149,6 +149,8 @@ pub struct Registry {
     // highlight. To do that we had to check the end regex in the tokenizer separately from the
     // regset.
     matcher_cache: papaya::HashMap<(GrammarId, GlobalRuleRef), Arc<RuleMatcher>>,
+    // whether to walk or only use regset
+    match_strategy: MatchStrategy,
 }
 
 impl Clone for Registry {
@@ -162,6 +164,7 @@ impl Clone for Registry {
             linked: self.linked,
             regex_cache: self.regex_cache.clone(),
             matcher_cache: papaya::HashMap::new(),
+            match_strategy: self.match_strategy,
         }
     }
 }
@@ -198,6 +201,7 @@ impl Registry {
             linked: false,
             regex_cache: Arc::new(RegexCache::default()),
             matcher_cache: papaya::HashMap::new(),
+            match_strategy: MatchStrategy::default(),
         };
         this.link_grammars();
 
@@ -321,6 +325,12 @@ impl Registry {
         Ok((tokens, tokenizer.into_scope_interner()))
     }
 
+    /// See [MatchStrategy] documentation to see which one to use for your usecase
+    pub fn set_match_strategy(&mut self, match_strategy: MatchStrategy) {
+        self.match_strategy = match_strategy;
+        self.clear_matcher_cache();
+    }
+
     /// Checks whether the given lang is available in the registry with its grammar name
     /// or aliases
     pub fn contains_grammar(&self, name: &str) -> bool {
@@ -554,10 +564,11 @@ impl Registry {
     pub fn clear_caches(&self) {
         self.matcher_cache.pin().clear();
         self.regex_cache.clear();
+        self.matcher_cache.pin().clear();
     }
 
     #[doc(hidden)]
-    pub fn clear_pattern_cache(&self) {
+    pub fn clear_matcher_cache(&self) {
         self.matcher_cache.pin().clear();
     }
 
@@ -578,7 +589,11 @@ impl Registry {
             .into_iter()
             .map(|(rule, pat)| (rule, pat.to_owned()))
             .collect();
-        let rule_matcher = Arc::new(RuleMatcher::new(patterns, self.regex_cache.clone()));
+        let rule_matcher = Arc::new(RuleMatcher::new(
+            patterns,
+            self.regex_cache.clone(),
+            self.match_strategy,
+        ));
 
         // Use get_or_insert for concurrent-safe lazy init
         let inserted = self