Fix #236: treat max_tokens=0 as empty string (#356)

`max_tokens=0` was compiled into a token-limited lexeme that never
terminates at runtime (the `{`-then-garbage symptom in #236). Since a
rule capped at zero tokens can only match the empty string, this
special-cases it to compile to `""`, placed before the
stop/terminal/subgrammar dispatch so it's handled uniformly for all body
types. Includes a regression test (`test_ll_max_tokens_zero`) with the
issue's repro plus a `name: ""` equivalence baseline. Full parser suite
+ fmt + clippy clean.

Author: pjdurden

parser/src/lark/compiler.rs

@@ -524,6 +524,17 @@ impl Compiler {
             bail!("max_tokens= is not supported for parametric rules");
         }
 
+        if rule.max_tokens == Some(0) {
+            // max_tokens=N caps a rule at N emitted tokens, so max_tokens=0
+            // forces zero emitted tokens: the rule can only match the empty
+            // string (epsilon). Compiling it as a token-limited lexeme (or
+            // subgrammar) instead produces broken runtime output -- the matcher
+            // emits an opening token and then never terminates. Treat it the
+            // same as an empty-string body `""`.
+            // See https://github.com/guidance-ai/llguidance/issues/236
+            return Ok(self.builder.string(""));
+        }
+
         let id = if let Some(stop) = rule.stop_like() {
             let is_suffix = rule.suffix.is_some();
             let is_empty = matches!(stop, Value::LiteralString(s, _) if s.is_empty());

parser/tests/test_ll.rs

@@ -680,6 +680,31 @@ fn test_ll_max_tokens() {
     );
 }
 
+#[test]
+fn test_ll_max_tokens_zero() {
+    // max_tokens=0 forces zero emitted tokens, i.e. the rule matches only the
+    // empty string -- so `name` contributes nothing and we go straight to "xy".
+    // See https://github.com/guidance-ai/llguidance/issues/236
+
+    // baseline: an explicitly empty rule works
+    check_lark_grammar(
+        r#"start: "Foo " num " x" name "y"
+           num: /[0-9]+/
+           name: ""
+        "#,
+        &["Foo‧ ", "5‧6‧ ", "xy"],
+    );
+
+    // the bug: max_tokens=0 should behave identically to name: ""
+    check_lark_grammar(
+        r#"start: "Foo " num " x" name "y"
+           num: /[0-9]+/
+           name[max_tokens=0]: /.*/
+        "#,
+        &["Foo‧ ", "5‧6‧ ", "xy"],
+    );
+}
+
 #[test]
 fn test_ll_special_token() {
     check_lark_grammar(