fix(lark): treat max_tokens=0 as empty string

A rule with max_tokens=0 should be allowed to emit no tokens, i.e. match
only the empty string. It was instead compiled to a token-limited lexeme
that never terminates at runtime (emits an opening token then loops).
Special-case max_tokens=0 to compile to "", uniformly across terminal,
subgrammar and json bodies. Adds a regression test.

Fixes #236

Author: pjdurden

parser/src/lark/compiler.rs

@@ -524,6 +524,16 @@ impl Compiler {
             bail!("max_tokens= is not supported for parametric rules");
         }
 
+        if rule.max_tokens == Some(0) {
+            // max_tokens=0 means the rule is allowed to emit no tokens at all,
+            // i.e. it matches only the empty string. Compiling it as a
+            // token-limited lexeme (or subgrammar) instead produces broken
+            // runtime output -- the matcher emits an opening token and then
+            // never terminates. Treat it the same as an empty-string body `""`.
+            // See https://github.com/guidance-ai/llguidance/issues/236
+            return Ok(self.builder.string(""));
+        }
+
         let id = if let Some(stop) = rule.stop_like() {
             let is_suffix = rule.suffix.is_some();
             let is_empty = matches!(stop, Value::LiteralString(s, _) if s.is_empty());

parser/tests/test_ll.rs

@@ -680,6 +680,31 @@ fn test_ll_max_tokens() {
     );
 }
 
+#[test]
+fn test_ll_max_tokens_zero() {
+    // max_tokens=0 means the rule emits no tokens, i.e. it matches the empty
+    // string -- so `name` contributes nothing and we go straight to "xy".
+    // See issue #236.
+
+    // baseline: an explicitly empty rule works
+    check_lark_grammar(
+        r#"start: "Foo " num " x" name "y"
+           num: /[0-9]+/
+           name: ""
+        "#,
+        &["Foo‧ ", "5‧6‧ ", "xy"],
+    );
+
+    // the bug: max_tokens=0 should behave identically to name: ""
+    check_lark_grammar(
+        r#"start: "Foo " num " x" name "y"
+           num: /[0-9]+/
+           name[max_tokens=0]: /.*/
+        "#,
+        &["Foo‧ ", "5‧6‧ ", "xy"],
+    );
+}
+
 #[test]
 fn test_ll_special_token() {
     check_lark_grammar(