From e7795f6d5db68752648aba6ae39665df070c9845 Mon Sep 17 00:00:00 2001 From: banana split Date: Mon, 1 Jun 2026 23:18:03 -0400 Subject: [PATCH 01/18] last_gentxt_size -> last_gentxt_len --- examples/server/server-context.cpp | 6 +++--- examples/server/server-context.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 4b1499f1fe..c4dfe618b3 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -607,7 +607,7 @@ void server_slot::prompt_load(server_prompt_cache& prompt_cache, const server_to void server_slot::reset() { n_prompt_tokens = 0; - last_gentxt_size = 0; + last_gentxt_len = 0; generated_text = ""; truncated = false; stopped_eos = false; @@ -2214,7 +2214,7 @@ bool server_context::process_token(completion_token_output& result, server_slot& slot.sampled = result.tok; // search stop word and delete it - slot.last_gentxt_size = slot.generated_text.size(); + slot.last_gentxt_len = slot.generated_text.size(); slot.generated_text += token_str; slot.has_next_token = true; @@ -4664,7 +4664,7 @@ void server_context::update_allowlist_state(server_slot& slot) { // search for keyword auto kw = kws[idx]; - auto pos = slot.generated_text.find(kw, std::max(0, slot.last_gentxt_size - (int32_t)kw.length() + 1)); + auto pos = slot.generated_text.find(kw, std::max(0, slot.last_gentxt_len - (int32_t)kw.length() + 1)); while (pos != std::string::npos) { if (++idx >= kws.size()) { break; diff --git a/examples/server/server-context.h b/examples/server/server-context.h index a33c2113b9..b9f7e8f8c2 100644 --- a/examples/server/server-context.h +++ b/examples/server/server-context.h @@ -75,7 +75,7 @@ struct server_slot { server_tokens prompt_tokens; server_tokens cache_tokens; - int32_t last_gentxt_size = 0; + int32_t last_gentxt_len = 0; std::string generated_text; // idx of draft tokens in the main batch From 8c2b69743c76ac5f5e2b4c545c59898e85fd1e39 Mon Sep 17 00:00:00 2001 From: banana split Date: Sat, 6 Jun 2026 14:41:11 -0400 Subject: [PATCH 02/18] reorg and delete unused member --- common/common.cpp | 8 -------- common/sampling.cpp | 20 ++++++++++---------- common/sampling.h | 13 ++++++------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 224f3e0df7..16d5c98ca0 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -5421,18 +5421,10 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp } } - size_t max_phrase_len = 0; - for (const auto& phrase: phrases) { - LLAMA_LOG_DEBUG("%s: line %zu: phrase = \"%s\"\n", __func__, i, phrase.c_str()); - max_phrase_len = std::max(phrase.length(), max_phrase_len); - } - LLAMA_LOG_DEBUG("%s: line %zu: max_phrase_len = %zu\n", __func__, i, max_phrase_len); - common_params_sampling::elb_param::elb_entry entry = { std::vector(n_phrase, 0), std::move(addsubs), std::vector(n_phrase, false), - max_phrase_len, std::move(phrases), std::move(biases), duration, diff --git a/common/sampling.cpp b/common/sampling.cpp index 03504beeee..006af11dd1 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -827,17 +827,17 @@ static void elb_sub(common_params_sampling& sparams, const common_params_samplin } void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float* logits) { - auto index_first_inactive = [](auto countup, auto& tokens) { + const auto& elb = ctx_sampling->elb_states[ctx_sampling->elb_idx]; + + auto index_first_inactive = [&elb](auto& tokens) { return std::distance( tokens.begin(), - std::upper_bound(tokens.begin(), tokens.end(), countup, [](const auto& countup, const auto& token) { - return countup > token.duration; + std::upper_bound(tokens.begin(), tokens.end(), NULL, [&elb](const auto& _, const auto& token) { + return elb.countup > token.duration; }) ); }; - const auto& elb = ctx_sampling->elb_states[ctx_sampling->elb_idx]; - std::string combined_text; const std::string* search_window = &combined_text; if (!ctx_sampling->drafted_text.empty()) { @@ -850,7 +850,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } if (!search_window->empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { - const auto ifi = index_first_inactive(elb.countup, elb.other_tokens); + const auto ifi = index_first_inactive(elb.other_tokens); for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.other_tokens[j]; if (string_ends_with(*search_window, cond)) { @@ -860,7 +860,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } if (!elb.first_tokens.empty() && (elb.first_tokens.front().duration > elb.countup)) { - const auto ifi = index_first_inactive(elb.countup, elb.first_tokens); + const auto ifi = index_first_inactive(elb.first_tokens); if (search_window->empty()) { // empty case here for (size_t j = 0; j < ifi; ++j) { @@ -908,13 +908,13 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float continue; // next entry } size_t count = 0; - auto pos = ctx_sampling->to_generated_text->find(phrase, entry.posi[j]); + auto pos = ctx_sampling->to_generated_text->find(phrase, entry.search_posi[j]); while (pos != std::string::npos) { LLAMA_LOG_DEBUG("%s: found %s @ %zu\n", __func__, phrase.c_str(), pos); ++count; pos = ctx_sampling->to_generated_text->find(phrase, pos + phrase.length()); } - entry.posi[j] = std::max(0, int32_t(ctx_sampling->to_generated_text->length()) - int32_t(phrase.length()) + 1); + entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->to_generated_text->length()) - int32_t(phrase.length()) + 1); if (count % 2 == 1) { // even = no match or cancelled LLAMA_LOG_DEBUG("%s: before\n", __func__); @@ -982,7 +982,7 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru // prepare next sampler bias for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) { // no clearance for sampler bias - std::fill(entry.posi.begin(), entry.posi.end(), pos); + std::fill(entry.search_posi.begin(), entry.search_posi.end(), pos); } } diff --git a/common/sampling.h b/common/sampling.h index c36aff4f40..4b888b7af7 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -174,14 +174,13 @@ typedef struct common_params_sampling { std::vector penalty_prompt_tokens; bool use_penalty_prompt_tokens = false; - // expiring logit bias + // expiring logit bias (elb) + expiring sparam bias (epb) struct elb_param { struct elb_entry { - std::vector posi; // positions of phrases in generated text - std::vector addsubs; // add/modify then subtract/restore sampling parameters + std::vector search_posi; // epb: starting search positions for phrases + std::vector addsubs; // epb: bias for sparams std::vector addflags; // true if added - size_t max_phrase_len; - std::vector phrases; + std::vector phrases; // elb: exitwords OR epb: keywords std::vector biases; // for each phrase, nth bias for nth token, extrapolate int32_t duration; // bias duration, unless exitword matches bool is_range; // has lower and upper biases @@ -192,7 +191,7 @@ typedef struct common_params_sampling { && (phrases == other.phrases) && (addflags == other.addflags) && (addsubs == other.addsubs) - && (posi == other.posi); + && (search_posi == other.search_posi); } }; std::vector entries; @@ -262,7 +261,7 @@ struct common_sampler { }; std::vector elb_states; size_t elb_idx; // for elb_states - size_t elb_search_pos; + size_t elb_search_pos; // for exitwords }; From cddda17a5f2b7c3eb4cef0ee5fd701bf19552794 Mon Sep 17 00:00:00 2001 From: banana split Date: Sun, 7 Jun 2026 12:47:35 -0400 Subject: [PATCH 03/18] handle rewind, simplify logic --- common/sampling.cpp | 143 ++++++++++++++++++++--------- common/sampling.h | 28 +++--- examples/server/server-context.cpp | 19 ++-- 3 files changed, 126 insertions(+), 64 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 006af11dd1..9f168149b6 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -175,6 +175,8 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co } } + result->n_rewind = 0; + result->elb_idx = 0; result->elb_search_pos = 0; @@ -226,6 +228,10 @@ void common_sampler_review(common_sampler * ctx, const size_t n_unsent, const bo if (ctx->adapt_p_ctx != nullptr) { llama_review_adaptive_p(ctx->adapt_p_ctx, n_unsent, rewind_status); } + + if (ctx->n_rewind > 0) { + common_expiring_logit_bias_rewind(ctx); + } } void llama_sampling_set_rng_seed(struct common_sampler * ctx, uint32_t seed) { @@ -717,6 +723,8 @@ void common_sampler_accept( } ctx_sampling->prev.push_back(token); + ctx_sampling->decoded_text += common_token_to_piece(ctx_main, token, true); + // grammar_should_apply() checks the reasoning budget state, so calculate this before we accept const auto accept_grammar = is_generated && grammar_should_apply(ctx_sampling); if (ctx_sampling->rbudget && is_generated) { @@ -730,7 +738,7 @@ void common_sampler_accept( llama_sampler_dry_accept(ctx_sampling->smpl, token); } - if (ctx_sampling->elb_states.size() > ctx_sampling->elb_idx) { + if (ctx_sampling->decoded_text.length() > ctx_sampling->elb_search_pos) { common_expiring_logit_bias_accept(ctx_sampling, ctx_main); } } @@ -779,8 +787,6 @@ std::vector common_sampler_sample_and_accept_n(struct common_sample for (; i < draft.size(); i++) { const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first); - gsmpl->drafted_text += common_token_to_piece(ctx, id, true); - common_sampler_accept(gsmpl, ctx, id, true); result.push_back(id); @@ -793,8 +799,6 @@ std::vector common_sampler_sample_and_accept_n(struct common_sample if (i == draft.size()) { const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first); - gsmpl->drafted_text += common_token_to_piece(ctx, id, true); - common_sampler_accept(gsmpl, ctx, id, true); result.push_back(id); @@ -827,6 +831,7 @@ static void elb_sub(common_params_sampling& sparams, const common_params_samplin } void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float* logits) { + // printf("%s[%d]: \n", __func__, __LINE__); const auto& elb = ctx_sampling->elb_states[ctx_sampling->elb_idx]; auto index_first_inactive = [&elb](auto& tokens) { @@ -838,22 +843,13 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float ); }; - std::string combined_text; - const std::string* search_window = &combined_text; - if (!ctx_sampling->drafted_text.empty()) { - // add speculated tokens - combined_text = ctx_sampling->to_generated_text != nullptr ? ( - ctx_sampling->to_generated_text->substr(std::max(0, int32_t(ctx_sampling->to_generated_text->length()) - elb.max_cond_len)) - ) : "" + ctx_sampling->drafted_text; - } else if (ctx_sampling->to_generated_text != nullptr) { - search_window = ctx_sampling->to_generated_text; - } + const std::string window = ctx_sampling->decoded_text.substr(ctx_sampling->elb_search_pos); - if (!search_window->empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { + if (!window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.other_tokens); for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.other_tokens[j]; - if (string_ends_with(*search_window, cond)) { + if (string_ends_with(window, cond)) { logits[id] += bias; } } @@ -861,7 +857,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float if (!elb.first_tokens.empty() && (elb.first_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.first_tokens); - if (search_window->empty()) { + if (window.empty()) { // empty case here for (size_t j = 0; j < ifi; ++j) { logits[elb.first_tokens[j].id] += elb.first_tokens[j].bias; @@ -870,7 +866,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.first_tokens[j]; // no bias if seen (probably too late) - if (!string_ends_with(*search_window, cond)) { + if (!string_ends_with(window, cond)) { logits[id] += bias; } } @@ -908,13 +904,13 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float continue; // next entry } size_t count = 0; - auto pos = ctx_sampling->to_generated_text->find(phrase, entry.search_posi[j]); + auto pos = ctx_sampling->decoded_text.find(phrase, entry.search_posi[j]); while (pos != std::string::npos) { LLAMA_LOG_DEBUG("%s: found %s @ %zu\n", __func__, phrase.c_str(), pos); ++count; - pos = ctx_sampling->to_generated_text->find(phrase, pos + phrase.length()); + pos = ctx_sampling->decoded_text.find(phrase, pos + phrase.length()); } - entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->to_generated_text->length()) - int32_t(phrase.length()) + 1); + entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->decoded_text.length()) - int32_t(phrase.length()) + 1); if (count % 2 == 1) { // even = no match or cancelled LLAMA_LOG_DEBUG("%s: before\n", __func__); @@ -928,41 +924,37 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } } } + // printf("%s[%d]: \n", __func__, __LINE__); } void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, struct llama_context * ctx_main) { - if (ctx_sampling->to_generated_text == nullptr) { - // prompt processing + // printf("%s[%d]: \n", __func__, __LINE__); + const auto idx = ctx_sampling->elb_idx; + if (idx >= ctx_sampling->elb_states.size()) { + ctx_sampling->elb_idx++; return; } - auto idx = ctx_sampling->elb_idx; auto& elb = ctx_sampling->elb_states[idx]; - if ((elb.delay > ++elb.countup) || (elb.search_word_len == 0)) { + if ((elb.delay > ++(elb.countup)) || (elb.search_word_len == 0)) { return; } - const std::string window = ctx_sampling->to_generated_text->substr(std::min( - ctx_sampling->to_generated_text->length(), - ctx_sampling->elb_search_pos)) + common_token_to_piece(ctx_main, ctx_sampling->prev.back(), true); + auto& search_pos = ctx_sampling->elb_search_pos; + const std::string window = ctx_sampling->decoded_text.substr(search_pos); + size_t pos = 0; if (string_is_found(window, elb.jumpword, pos)) { - LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(window).c_str(), pos); - pos += ctx_sampling->elb_search_pos + elb.jumpword.length(); + LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(window).c_str(), search_pos + pos); ctx_sampling->elb_idx = elb.jump_idx; } else if (string_is_found(window, elb.exitword, pos)) { - LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(window).c_str(), pos); - pos += ctx_sampling->elb_search_pos + elb.exitword.length(); - ++ctx_sampling->elb_idx; + LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(window).c_str(), search_pos + pos); + ctx_sampling->elb_idx++; } else { - // not found. move search position to include next token - ctx_sampling->elb_search_pos += std::max(0, int32_t(window.length()) - int32_t(elb.search_word_len) + 1); + search_pos += std::max(0, int32_t(window.length()) - elb.search_word_len); return; } - - // single character clearance - // e.g. stop \n\n from expiring two \n immediately - ctx_sampling->elb_search_pos = pos + 1; + search_pos += pos; // undo current sampler bias for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { @@ -981,9 +973,76 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru // prepare next sampler bias for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) { - // no clearance for sampler bias - std::fill(entry.search_posi.begin(), entry.search_posi.end(), pos); + entry.search_posi.assign(entry.search_posi.size(), search_pos); + } + + // single character clearance for exitword + search_pos++; + + ctx_sampling->elb_states[ctx_sampling->elb_idx].init_pos = search_pos; + // printf("%s[%d]: \n", __func__, __LINE__); +} + +void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { + printf("%s[%d]: \n", __func__, __LINE__); + auto& idx = ctx_sampling->elb_idx; + auto n_rewind = ctx_sampling->n_rewind; + + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + + if (idx >= ctx_sampling->elb_states.size()) { + // entirely expired + n_rewind -= 1 + idx - ctx_sampling->elb_states.size(); + idx = ctx_sampling->elb_states.size() - 1; + if (n_rewind <= 0) { + // not enough rewind to reanimate + idx -= n_rewind; + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + return; + } } + + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + + // consume n_rewind + while (n_rewind > 0) { + while (ctx_sampling->elb_states[idx].countup == 0) { + --idx; + } + auto& countup = ctx_sampling->elb_states[idx].countup; + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, n_rewind = %d\n", __func__, __LINE__, idx, countup, n_rewind); + if (countup > n_rewind) { + countup -= n_rewind; + n_rewind = 0; + } else { + n_rewind -= countup; + countup = 0; + } + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, n_rewind = %d\n", __func__, __LINE__, idx, countup, n_rewind); + } + + // rewind sparam bias including new current + for (int32_t j = idx; j < ctx_sampling->elb_states.size(); ++j) { + printf("%s[%d]: \n", __func__, __LINE__); + for (auto& entry: ctx_sampling->params.elb_params[j].entries) { + printf("%s[%d]: %zu\n", __func__, __LINE__, entry.addflags.size()); + for (const auto addflag: entry.addflags) { + printf("%s[%d]: \n", __func__, __LINE__); + if (addflag) { + LLAMA_LOG_DEBUG("%s: before\n", __func__); + elb_print(ctx_sampling->params, entry); + + elb_sub(ctx_sampling->params, entry); + + LLAMA_LOG_DEBUG("%s: after\n", __func__); + elb_print(ctx_sampling->params, entry); + } + } + } + } + + ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos; + LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos); } diff --git a/common/sampling.h b/common/sampling.h index 4b888b7af7..ecf97b3756 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -177,13 +177,13 @@ typedef struct common_params_sampling { // expiring logit bias (elb) + expiring sparam bias (epb) struct elb_param { struct elb_entry { - std::vector search_posi; // epb: starting search positions for phrases + std::vector search_posi; // epb: starting search positions for phrases std::vector addsubs; // epb: bias for sparams - std::vector addflags; // true if added + std::vector addflags; // epb: true if added std::vector phrases; // elb: exitwords OR epb: keywords - std::vector biases; // for each phrase, nth bias for nth token, extrapolate - int32_t duration; // bias duration, unless exitword matches - bool is_range; // has lower and upper biases + std::vector biases; // for each phrase, nth bias for nth token, extrapolate + int32_t duration; // bias duration, unless exitword matches + bool is_range; // has lower and upper biases bool operator == (const struct elb_entry& other) const { return (is_range == other.is_range) && (duration == other.duration) @@ -232,14 +232,15 @@ struct common_sampler { size_t n_valid; // Number of correct top tokens with correct probabilities. + int32_t n_rewind; + llama_token_data_array cur_p; // current candidates std::mt19937 rng; std::vector* server_biases; - std::string drafted_text; - std::string* to_generated_text = nullptr; + std::string decoded_text; // expiring logit bias struct elb_state { @@ -247,7 +248,7 @@ struct common_sampler { int32_t id; float bias; size_t duration; - std::string cond; // bias activation condition + std::string cond; // bias activation condition }; std::vector first_tokens; // first token of each phrase std::vector other_tokens; @@ -256,12 +257,13 @@ struct common_sampler { size_t delay; // to avoid early termination of positively biased phrases int32_t max_cond_len; std::string jumpword; - size_t jump_idx; - size_t search_word_len; + int32_t jump_idx; + int32_t search_word_len; + int32_t init_pos; }; std::vector elb_states; - size_t elb_idx; // for elb_states - size_t elb_search_pos; // for exitwords + int32_t elb_idx; // for elb_states + int32_t elb_search_pos; // for exitwords }; @@ -363,5 +365,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, struct llama_context * ctx_main); +void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling); + llama_grammar* llama_sampler_init_llg(const llama_vocab* vocab, const char* grammar_kind, const char* grammar_data); diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index c4dfe618b3..df80588a1b 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -1940,6 +1940,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) common_sampler_free(slot.ctx_sampling); } slot.ctx_sampling = common_sampler_init(model, slot.sparams); + slot.ctx_sampling->decoded_text.reserve(4 * slot.n_ctx); } catch (std::exception & e) { std::string err_msg = std::string("Failed to initialize samplers: ") + e.what(); @@ -2009,7 +2010,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) } } - slot.ctx_sampling->elb_states.push_back({ { }, { }, exitword, 0, 0, 0, "", 0, exitword.length() }); + slot.ctx_sampling->elb_states.push_back({ { }, { }, exitword, 0, 0, 0, "", 0, int32_t(exitword.length()), 0 }); auto& first_tokens = slot.ctx_sampling->elb_states.back().first_tokens; auto& other_tokens = slot.ctx_sampling->elb_states.back().other_tokens; @@ -4270,13 +4271,6 @@ void server_context::speculative_decoding_accept() { const common_speculative_type spec_type_used = slot.drafted_spec_type; size_t n_draft = slot.drafted.size(); - slot.ctx_sampling->to_generated_text = &slot.generated_text; - if (n_draft > 0) { - (void) populate_vocab_pieces(); // max_piece_len - slot.ctx_sampling->drafted_text.reserve(max_piece_len * n_draft); - slot.ctx_sampling->drafted_text.clear(); - } - apply_server_biases(slot); // the accepted tokens from the speculation @@ -4375,6 +4369,8 @@ void server_context::speculative_decoding_accept() { populate_token_probs(slot, result, slot.params.post_sampling_probs, params_base.special, i); } + slot.ctx_sampling->n_rewind = 0; + if (slot.n_buffer == 0 || !params_base.can_ban_phrases) { if (!process_token(result, slot)) { // release slot because of stop condition @@ -4540,6 +4536,9 @@ inline void rewind_context(server_slot& slot, int32_t ban_pos) { int32_t n_keep_buffer = ban_pos - buffer_start_pos; if (n_keep_buffer < 0) n_keep_buffer = 0; + slot.ctx_sampling->n_rewind = slot.token_buffer.size() - n_keep_buffer; + LLAMA_LOG_DEBUG("%s: rewinding %d tokens\n", __func__, slot.ctx_sampling->n_rewind); + if (slot.banned_n != 0) { int32_t n = 0; for (auto result = slot.token_buffer.begin() + n_keep_buffer; result != slot.token_buffer.end(); result++) { @@ -4797,8 +4796,6 @@ void server_context::process_batch_tokens(int32_t & n_batch) { } } - slot.ctx_sampling->to_generated_text = &slot.generated_text; - completion_token_output result; const int tok_idx = slot.i_batch - i; @@ -4851,6 +4848,8 @@ void server_context::process_batch_tokens(int32_t & n_batch) { populate_token_probs(slot, result, slot.params.post_sampling_probs, params_base.special, tok_idx); } + slot.ctx_sampling->n_rewind = 0; + // no ban string for recurrent/hybrid model if (slot.n_buffer == 0 || !params_base.can_ban_phrases) { slot.token_buffer = { result }; From e3b4a52c2e065f008d4d2a33f983d5c0c2528aa1 Mon Sep 17 00:00:00 2001 From: banana split Date: Sun, 7 Jun 2026 16:26:43 -0400 Subject: [PATCH 04/18] update rewind_context() to loop over all rewinded tokens --- common/sampling.h | 3 ++- examples/server/server-context.cpp | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/common/sampling.h b/common/sampling.h index ecf97b3756..b1c019a38b 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -232,7 +232,8 @@ struct common_sampler { size_t n_valid; // Number of correct top tokens with correct probabilities. - int32_t n_rewind; + int32_t n_rewind; + std::string rewinded_text; llama_token_data_array cur_p; // current candidates diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index df80588a1b..f69dc357f5 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -4370,6 +4370,7 @@ void server_context::speculative_decoding_accept() { } slot.ctx_sampling->n_rewind = 0; + slot.ctx_sampling->rewinded_text.clear(); if (slot.n_buffer == 0 || !params_base.can_ban_phrases) { if (!process_token(result, slot)) { @@ -4537,24 +4538,23 @@ inline void rewind_context(server_slot& slot, int32_t ban_pos) { if (n_keep_buffer < 0) n_keep_buffer = 0; slot.ctx_sampling->n_rewind = slot.token_buffer.size() - n_keep_buffer; + slot.ctx_sampling->rewinded_text.reserve(4 * slot.ctx_sampling->n_rewind); LLAMA_LOG_DEBUG("%s: rewinding %d tokens\n", __func__, slot.ctx_sampling->n_rewind); - if (slot.banned_n != 0) { - int32_t n = 0; - for (auto result = slot.token_buffer.begin() + n_keep_buffer; result != slot.token_buffer.end(); result++) { - llama_token banned_tok = result->tok; + for (int32_t n = 0; n < slot.ctx_sampling->n_rewind; ++n) { + const auto& result = slot.token_buffer.begin() + n_keep_buffer + n; + llama_token banned_tok = result->tok; + + if ((slot.banned_n < 0) || (n < slot.banned_n)) { + slot.positional_bans[ban_pos].insert(banned_tok); if (n == 0) { LLAMA_LOG_DEBUG("Banned pattern detected at pos %d. Banning token %d ('%s') and rewinding.\n", ban_pos, banned_tok, result->text_to_send.c_str()); } - - slot.positional_bans[ban_pos].insert(banned_tok); - n++; - if (slot.banned_n > 0 && n == slot.banned_n) { - break; - } } + + slot.ctx_sampling->rewinded_text.append(result->text_to_send); } int32_t n_rewind_total = (slot.n_past + 1) - ban_pos; @@ -4849,6 +4849,7 @@ void server_context::process_batch_tokens(int32_t & n_batch) { } slot.ctx_sampling->n_rewind = 0; + slot.ctx_sampling->rewinded_text.clear(); // no ban string for recurrent/hybrid model if (slot.n_buffer == 0 || !params_base.can_ban_phrases) { From 281f69c45d54f9154d13a97e55edd20da0ac8ebe Mon Sep 17 00:00:00 2001 From: banana split Date: Sun, 7 Jun 2026 20:04:17 -0400 Subject: [PATCH 05/18] add special rewind case: idx=0 --- common/sampling.cpp | 90 ++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 9f168149b6..17882be091 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -831,7 +831,6 @@ static void elb_sub(common_params_sampling& sparams, const common_params_samplin } void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float* logits) { - // printf("%s[%d]: \n", __func__, __LINE__); const auto& elb = ctx_sampling->elb_states[ctx_sampling->elb_idx]; auto index_first_inactive = [&elb](auto& tokens) { @@ -850,6 +849,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.other_tokens[j]; if (string_ends_with(window, cond)) { + // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id); logits[id] += bias; } } @@ -860,6 +860,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float if (window.empty()) { // empty case here for (size_t j = 0; j < ifi; ++j) { + // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, elb.first_tokens[j].id); logits[elb.first_tokens[j].id] += elb.first_tokens[j].bias; } } else { @@ -867,13 +868,14 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float const auto& [id, bias, _, cond] = elb.first_tokens[j]; // no bias if seen (probably too late) if (!string_ends_with(window, cond)) { + // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id); logits[id] += bias; } } } } - // expiring sampler bias + // expiring sparam bias for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) { if (!entry.biases.empty()) { continue; // next entry @@ -924,11 +926,9 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } } } - // printf("%s[%d]: \n", __func__, __LINE__); } void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, struct llama_context * ctx_main) { - // printf("%s[%d]: \n", __func__, __LINE__); const auto idx = ctx_sampling->elb_idx; if (idx >= ctx_sampling->elb_states.size()) { ctx_sampling->elb_idx++; @@ -936,6 +936,7 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru } auto& elb = ctx_sampling->elb_states[idx]; + // LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, %s\n", __func__, __LINE__, idx, elb.countup, string_unescape(common_token_to_piece(ctx_main, ctx_sampling->prev.back(), true)).c_str()); if ((elb.delay > ++(elb.countup)) || (elb.search_word_len == 0)) { return; } @@ -947,16 +948,17 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru if (string_is_found(window, elb.jumpword, pos)) { LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(window).c_str(), search_pos + pos); ctx_sampling->elb_idx = elb.jump_idx; + search_pos += pos + elb.jumpword.length(); } else if (string_is_found(window, elb.exitword, pos)) { LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(window).c_str(), search_pos + pos); ctx_sampling->elb_idx++; + search_pos += pos + elb.exitword.length(); } else { search_pos += std::max(0, int32_t(window.length()) - elb.search_word_len); return; } - search_pos += pos; - // undo current sampler bias + // undo current sparam bias for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { for (const auto addflag: entry.addflags) { if (addflag) { @@ -971,24 +973,18 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru } } - // prepare next sampler bias + // initialize next stage + ctx_sampling->elb_states[ctx_sampling->elb_idx].init_pos = search_pos; for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) { entry.search_posi.assign(entry.search_posi.size(), search_pos); } - - // single character clearance for exitword - search_pos++; - - ctx_sampling->elb_states[ctx_sampling->elb_idx].init_pos = search_pos; - // printf("%s[%d]: \n", __func__, __LINE__); } void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { - printf("%s[%d]: \n", __func__, __LINE__); auto& idx = ctx_sampling->elb_idx; auto n_rewind = ctx_sampling->n_rewind; - LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + // LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); if (idx >= ctx_sampling->elb_states.size()) { // entirely expired @@ -997,12 +993,11 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { if (n_rewind <= 0) { // not enough rewind to reanimate idx -= n_rewind; - LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); return; } } - LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + // LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); // consume n_rewind while (n_rewind > 0) { @@ -1020,14 +1015,62 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, n_rewind = %d\n", __func__, __LINE__, idx, countup, n_rewind); } + LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + + if (idx == 0) { + // trivial case + const auto& decoded_text = ctx_sampling->decoded_text; + auto& elb = ctx_sampling->elb_states[idx]; + std::string window; + for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { + if (!entry.biases.empty()) { + // no need to rewind elb + continue; + } + + for (size_t j = 0; j < entry.phrases.size(); ++j) { + const auto& phrase = entry.phrases[j]; + if (phrase.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { + elb_add(ctx_sampling->params, entry); + entry.addflags[j] = true; + continue; + } + + std::string_view decoded_tail; + if (phrase.length() > decoded_text.length()) { + decoded_tail = decoded_text; + entry.search_posi[j] = 0; + } else { + decoded_tail = decoded_text.substr(decoded_text.length() - phrase.length() + 1); + entry.search_posi[j] = decoded_text.length() - phrase.length() + 1; + } + window.clear(); + window.append(decoded_tail); + window.append(ctx_sampling->rewinded_text); + + // triggered within rewinded window? + size_t count = 0; + auto pos = window.find(phrase, entry.search_posi[j]); + while (pos != std::string::npos) { + ++count; + pos = window.find(phrase, pos + phrase.length()); + } + if (count % 2 == 1) { + (entry.addflags[j] ? elb_sub : elb_add)(ctx_sampling->params, entry); + entry.addflags[j] = !entry.addflags[j]; + } + } + } + ctx_sampling->elb_search_pos = std::max(0, int32_t(decoded_text.length()) - elb.search_word_len + 1); + } else { + ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos; + } + LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos); - // rewind sparam bias including new current - for (int32_t j = idx; j < ctx_sampling->elb_states.size(); ++j) { - printf("%s[%d]: \n", __func__, __LINE__); + // rewind sparam bias + for (int32_t j = std::max(1, idx); j < ctx_sampling->elb_states.size(); ++j) { for (auto& entry: ctx_sampling->params.elb_params[j].entries) { - printf("%s[%d]: %zu\n", __func__, __LINE__, entry.addflags.size()); for (const auto addflag: entry.addflags) { - printf("%s[%d]: \n", __func__, __LINE__); if (addflag) { LLAMA_LOG_DEBUG("%s: before\n", __func__); elb_print(ctx_sampling->params, entry); @@ -1040,9 +1083,6 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } } } - - ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos; - LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos); } From 2421f37c019e5b86c6f1713e77469154b8904dff Mon Sep 17 00:00:00 2001 From: banana split Date: Sun, 7 Jun 2026 20:54:05 -0400 Subject: [PATCH 06/18] restore rewind_context() logic --- examples/server/server-context.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index f69dc357f5..ad2f44b383 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -4540,21 +4540,26 @@ inline void rewind_context(server_slot& slot, int32_t ban_pos) { slot.ctx_sampling->n_rewind = slot.token_buffer.size() - n_keep_buffer; slot.ctx_sampling->rewinded_text.reserve(4 * slot.ctx_sampling->n_rewind); LLAMA_LOG_DEBUG("%s: rewinding %d tokens\n", __func__, slot.ctx_sampling->n_rewind); + for (int32_t j = n_keep_buffer; j < slot.token_buffer.size(); ++j) { + slot.ctx_sampling->rewinded_text.append(slot.token_buffer[j].text_to_send); + } - for (int32_t n = 0; n < slot.ctx_sampling->n_rewind; ++n) { - const auto& result = slot.token_buffer.begin() + n_keep_buffer + n; - llama_token banned_tok = result->tok; - - if ((slot.banned_n < 0) || (n < slot.banned_n)) { - slot.positional_bans[ban_pos].insert(banned_tok); + if (slot.banned_n != 0) { + int32_t n = 0; + for (auto result = slot.token_buffer.begin() + n_keep_buffer; result != slot.token_buffer.end(); result++) { + llama_token banned_tok = result->tok; if (n == 0) { LLAMA_LOG_DEBUG("Banned pattern detected at pos %d. Banning token %d ('%s') and rewinding.\n", ban_pos, banned_tok, result->text_to_send.c_str()); } - } - slot.ctx_sampling->rewinded_text.append(result->text_to_send); + slot.positional_bans[ban_pos].insert(banned_tok); + n++; + if (slot.banned_n > 0 && n == slot.banned_n) { + break; + } + } } int32_t n_rewind_total = (slot.n_past + 1) - ban_pos; From 757db7d3c4c2e5c968e49542e251eedeebc4c736 Mon Sep 17 00:00:00 2001 From: banana split Date: Thu, 11 Jun 2026 21:42:23 -0400 Subject: [PATCH 07/18] simplify logic --- common/sampling.cpp | 75 +++++++++++++++++++-------------------------- common/sampling.h | 5 +-- 2 files changed, 34 insertions(+), 46 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 17882be091..498d53872b 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -1017,58 +1017,45 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); - if (idx == 0) { - // trivial case - const auto& decoded_text = ctx_sampling->decoded_text; - auto& elb = ctx_sampling->elb_states[idx]; - std::string window; - for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { - if (!entry.biases.empty()) { - // no need to rewind elb + const auto& rewinded_text = ctx_sampling->rewinded_text; + const auto& decoded_text = ctx_sampling->decoded_text; + auto& elb = ctx_sampling->elb_states[idx]; + for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { + if (!entry.biases.empty()) { + // no need to rewind elb + continue; + } + + for (size_t j = 0; j < entry.phrases.size(); ++j) { + const auto& phrase = entry.phrases[j]; + if (phrase.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { + elb_add(ctx_sampling->params, entry); + entry.addflags[j] = true; continue; } - for (size_t j = 0; j < entry.phrases.size(); ++j) { - const auto& phrase = entry.phrases[j]; - if (phrase.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { - elb_add(ctx_sampling->params, entry); - entry.addflags[j] = true; - continue; - } - - std::string_view decoded_tail; - if (phrase.length() > decoded_text.length()) { - decoded_tail = decoded_text; - entry.search_posi[j] = 0; - } else { - decoded_tail = decoded_text.substr(decoded_text.length() - phrase.length() + 1); - entry.search_posi[j] = decoded_text.length() - phrase.length() + 1; - } - window.clear(); - window.append(decoded_tail); - window.append(ctx_sampling->rewinded_text); - - // triggered within rewinded window? - size_t count = 0; - auto pos = window.find(phrase, entry.search_posi[j]); - while (pos != std::string::npos) { - ++count; - pos = window.find(phrase, pos + phrase.length()); - } - if (count % 2 == 1) { - (entry.addflags[j] ? elb_sub : elb_add)(ctx_sampling->params, entry); - entry.addflags[j] = !entry.addflags[j]; - } + // triggered within rewinded text? + size_t count = 0; + auto pos = decoded_text.find(phrase, std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - SSIZE(phrase) + 1)); + while (pos != std::string::npos) { + ++count; + pos = decoded_text.find(phrase, pos + 1); + } + if (count % 2 == 1) { + (entry.addflags[j] ? elb_sub : elb_add)(ctx_sampling->params, entry); + entry.addflags[j] = !entry.addflags[j]; } } - ctx_sampling->elb_search_pos = std::max(0, int32_t(decoded_text.length()) - elb.search_word_len + 1); - } else { - ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos; } - LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos); + ctx_sampling->elb_search_pos = std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - elb.search_word_len + 1); + + LLAMA_LOG_DEBUG("%s[%d]:\elb_search_pos = %d, initpos %d, dtlen %d, rtlen %d, swlen %d\n", __func__, __LINE__, + ctx_sampling->elb_search_pos, elb.init_pos, SSIZE(decoded_text), SSIZE(rewinded_text), elb.search_word_len); + + LLAMA_LOG_DEBUG("%s[%d]: %s\n", __func__, __LINE__, decoded_text.c_str()); // rewind sparam bias - for (int32_t j = std::max(1, idx); j < ctx_sampling->elb_states.size(); ++j) { + for (int32_t j = idx + 1; j < ctx_sampling->elb_states.size(); ++j) { for (auto& entry: ctx_sampling->params.elb_params[j].entries) { for (const auto addflag: entry.addflags) { if (addflag) { diff --git a/common/sampling.h b/common/sampling.h index b1c019a38b..0e29e52ee3 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -11,6 +11,8 @@ #define A_DOT_B(a, b) a.b +#define SSIZE(container) int32_t(container.size()) + // sampler types enum class llama_sampler_type : char { DRY = 'd', @@ -234,6 +236,7 @@ struct common_sampler { int32_t n_rewind; std::string rewinded_text; + std::string decoded_text; llama_token_data_array cur_p; // current candidates @@ -241,8 +244,6 @@ struct common_sampler { std::vector* server_biases; - std::string decoded_text; - // expiring logit bias struct elb_state { struct elb_token { From 646cebfeb8899b9f4a066f04a0b1777e0790964c Mon Sep 17 00:00:00 2001 From: banana split Date: Thu, 11 Jun 2026 22:46:12 -0400 Subject: [PATCH 08/18] add flag for marking end of prompt processing --- common/sampling.cpp | 13 +++++++++---- common/sampling.h | 2 ++ examples/server/server-context.cpp | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 498d53872b..1a1f07851b 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -175,6 +175,8 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co } } + result->is_decoding = false; + result->n_rewind = 0; result->elb_idx = 0; @@ -219,6 +221,7 @@ static void llama_grammar_reset(common_sampler * ctx) { void common_sampler_reset(common_sampler * ctx) { // llama_grammar_reset(ctx); + ctx->is_decoding = false; ctx->prev.clear(); llama_sampler_dry_reset(ctx->smpl); } @@ -723,8 +726,6 @@ void common_sampler_accept( } ctx_sampling->prev.push_back(token); - ctx_sampling->decoded_text += common_token_to_piece(ctx_main, token, true); - // grammar_should_apply() checks the reasoning budget state, so calculate this before we accept const auto accept_grammar = is_generated && grammar_should_apply(ctx_sampling); if (ctx_sampling->rbudget && is_generated) { @@ -738,8 +739,12 @@ void common_sampler_accept( llama_sampler_dry_accept(ctx_sampling->smpl, token); } - if (ctx_sampling->decoded_text.length() > ctx_sampling->elb_search_pos) { - common_expiring_logit_bias_accept(ctx_sampling, ctx_main); + if (ctx_sampling->is_decoding) { + ctx_sampling->decoded_text += common_token_to_piece(ctx_main, token, true); + + if (ctx_sampling->decoded_text.length() > ctx_sampling->elb_search_pos) { + common_expiring_logit_bias_accept(ctx_sampling, ctx_main); + } } } diff --git a/common/sampling.h b/common/sampling.h index 0e29e52ee3..9d2ab4dd5f 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -215,6 +215,8 @@ struct common_sampler { // parameters that will be used for sampling common_params_sampling params; + bool is_decoding; // set true near the end of pp + // mirostat sampler state float mirostat_mu; diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index ad2f44b383..7117048521 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -4104,6 +4104,7 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t GGML_ASSERT(batch.n_tokens > 0); GGML_ASSERT((size_t)slot.n_prompt_tokens == slot.prompt_tokens.size()); common_sampler_reset(slot.ctx_sampling); + slot.ctx_sampling->is_decoding = true; for (int i = 0; i < slot.n_prompt_tokens; ++i) { llama_token id = slot.prompt_tokens[i]; if (id != LLAMA_TOKEN_NULL) { From c89b09a07a7f51b83ab8efc9e8be58c85e57b12c Mon Sep 17 00:00:00 2001 From: banana split Date: Sat, 13 Jun 2026 23:15:16 -0400 Subject: [PATCH 09/18] revert some cosmetic changes --- common/sampling.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 1495c52110..0ef1ed5515 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -847,13 +847,13 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float ); }; - const std::string window = ctx_sampling->decoded_text.substr(ctx_sampling->elb_search_pos); + const std::string search_window = ctx_sampling->decoded_text.substr(ctx_sampling->elb_search_pos); - if (!window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { + if (!search_window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.other_tokens); for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.other_tokens[j]; - if (string_ends_with(window, cond)) { + if (string_ends_with(search_window, cond)) { // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id); logits[id] += bias; } @@ -862,7 +862,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float if (!elb.first_tokens.empty() && (elb.first_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.first_tokens); - if (window.empty()) { + if (search_window.empty()) { // empty case here for (size_t j = 0; j < ifi; ++j) { // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, elb.first_tokens[j].id); @@ -872,7 +872,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float for (size_t j = 0; j < ifi; ++j) { const auto& [id, bias, _, cond] = elb.first_tokens[j]; // no bias if seen (probably too late) - if (!string_ends_with(window, cond)) { + if (!string_ends_with(search_window, cond)) { // LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id); logits[id] += bias; } From 0c22838085e3ea1e4e615d0cee3d25b761d9a9b4 Mon Sep 17 00:00:00 2001 From: banana split Date: Tue, 16 Jun 2026 00:44:33 -0400 Subject: [PATCH 10/18] sampling: add ELB rewind support and optimize sampler text handling - Implements `common_expiring_logit_bias_rewind` to allow ELB/EPB state rollbacks during speculative decoding and banned phrase triggers. - Replaces `to_generated_text` pointer logic in `common_sampler` with `generated_text` view and `playing_text`. - Adds `string_assign_append` helper to optimize string window concatenation. - Hooks up `n_rewind` tracking in `server-context.cpp`. --- common/common.cpp | 25 +++++++-- common/common.h | 2 + common/sampling.cpp | 83 +++++++++++++++++++++--------- common/sampling.h | 30 ++++++----- examples/server/server-context.cpp | 13 ++++- 5 files changed, 112 insertions(+), 41 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 8ac72e3188..c356c745b0 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -3693,6 +3693,15 @@ bool string_is_found(const std::string& window, const std::string& str, size_t& return pos != std::string::npos; } +void string_assign_append(std::string& dst, const std::string_view& sv, const std::string& str, const int32_t pos) { + const int32_t margin = SSIZE(sv) - pos; + if (margin > 0) { + dst.assign(sv, pos).append(str); + } else { + dst.assign(str, -margin); + } +} + // // Filesystem utils // @@ -5388,7 +5397,7 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp static const std::vector names = { X_COMMON_PARAMS_SAMPLING }; std::vector addsubs(names.size(), 0.0f); - bool is_sb = false; + bool is_epb = false; // (... : SPARAM ...) const auto window = line.substr(last_qq_pos + 1); @@ -5404,7 +5413,7 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp auto sub = string_strip(window.substr(pos, next_pos - pos)); if (sub[0] == '~') { addsubs[j] += std::stof(sub.substr(1)); - is_sb = true; + is_epb = true; LLAMA_LOG_DEBUG("%s: line %zu: bias = %f\n", __func__, i, addsubs[j]); } } @@ -5412,18 +5421,19 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp auto& phrases = extracts; if (phrases.empty()) { - if (is_sb) { + if (is_epb) { phrases.push_back(""); } else { continue; // next line } } + int32_t max_keyword_len = 0; const auto n_phrase = phrases.size(); std::vector biases; bool is_range = false; - if (!is_sb) { + if (!is_epb) { // (... : BIAS ...) const auto cln_rpos = line.rfind(':'); auto sub = line.substr(cln_rpos + 1, n_char - cln_rpos - 2); @@ -5447,9 +5457,16 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp if (biases.empty()) { continue; // next line } + } else { + for (const auto& keyword: phrases) { + LLAMA_LOG_DEBUG("%s: line %zu: keyword = \"%s\"\n", __func__, i, keyword.c_str()); + max_keyword_len = std::max(SSIZE(keyword), max_keyword_len); + } + LLAMA_LOG_DEBUG("%s: line %zu: max_keyword_len = %zu\n", __func__, i, max_keyword_len); } common_params_sampling::elb_param::elb_entry entry = { + max_keyword_len, std::vector(n_phrase, 0), std::move(addsubs), std::vector(n_phrase, false), diff --git a/common/common.h b/common/common.h index 070d3b8a51..9a34f424a3 100644 --- a/common/common.h +++ b/common/common.h @@ -659,6 +659,8 @@ std::vector string_extract(const std::string& str, const char c, st bool string_is_found(const std::string& window, const std::string& str, size_t& pos); +void string_assign_append(std::string& dst, const std::string_view& sv, const std::string& str, const int32_t pos); + // // Filesystem utils // diff --git a/common/sampling.cpp b/common/sampling.cpp index 0ef1ed5515..5891a6de69 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -740,11 +740,9 @@ void common_sampler_accept( } if (ctx_sampling->is_decoding) { - ctx_sampling->decoded_text += common_token_to_piece(ctx_main, token, true); + ctx_sampling->playing_text += common_token_to_piece(ctx_main, token, true); - if (ctx_sampling->decoded_text.length() > ctx_sampling->elb_search_pos) { - common_expiring_logit_bias_accept(ctx_sampling, ctx_main); - } + common_expiring_logit_bias_accept(ctx_sampling, ctx_main); } } @@ -847,7 +845,11 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float ); }; - const std::string search_window = ctx_sampling->decoded_text.substr(ctx_sampling->elb_search_pos); + auto& search_window = ctx_sampling->scratch; + string_assign_append(search_window, + ctx_sampling->generated_text, + ctx_sampling->playing_text, + std::max(0, SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->generated_text) - elb.max_cond_len)); if (!search_window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.other_tokens); @@ -885,8 +887,10 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float if (!entry.biases.empty()) { continue; // next entry } - for (size_t j = 0; j < entry.phrases.size(); ++j) { - const auto& phrase = entry.phrases[j]; + + const auto& keywords = entry.phrases; + for (size_t j = 0; j < keywords.size(); ++j) { + const auto& phrase = keywords[j]; if (phrase.empty()) { // duration bound only if (elb.countup == 0) { @@ -910,14 +914,27 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } continue; // next entry } + + // search_window.clear(); + // margin = SSIZE(ctx_sampling->generated_text) - entry.search_posi[j]; + // if (margin > 0) { + // search_window.append(ctx_sampling->generated_text, entry.search_posi[j], -1).append(ctx_sampling->playing_text); + // } else { + // search_window.append(ctx_sampling->playing_text, -margin); + // } + string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, entry.search_posi[j]); + size_t count = 0; - auto pos = ctx_sampling->decoded_text.find(phrase, entry.search_posi[j]); + auto pos = search_window.find(phrase, entry.search_posi[j]); + // auto pos = ctx_sampling->decoded_text.find(phrase, entry.search_posi[j]); while (pos != std::string::npos) { LLAMA_LOG_DEBUG("%s: found %s @ %zu\n", __func__, phrase.c_str(), pos); ++count; - pos = ctx_sampling->decoded_text.find(phrase, pos + phrase.length()); + pos = search_window.find(phrase, pos + phrase.length()); + // pos = ctx_sampling->decoded_text.find(phrase, pos + phrase.length()); } - entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->decoded_text.length()) - int32_t(phrase.length()) + 1); + entry.search_posi[j] = SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(phrase) + 1; + // entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->decoded_text.length()) - int32_t(phrase.length()) + 1); if (count % 2 == 1) { // even = no match or cancelled LLAMA_LOG_DEBUG("%s: before\n", __func__); @@ -946,20 +963,30 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru return; } + // std::string window; + // int32_t margin = SSIZE(ctx_sampling->generated_text) - ctx_sampling->elb_search_pos; + // window.reserve(margin + SSIZE(ctx_sampling->playing_text)); + // if (margin > 0) { + // window.append(ctx_sampling->generated_text, ctx_sampling->elb_search_pos, -1).append(ctx_sampling->playing_text); + // } else { + // window.append(ctx_sampling->playing_text, -margin); + // } + auto& search_pos = ctx_sampling->elb_search_pos; - const std::string window = ctx_sampling->decoded_text.substr(search_pos); + auto& search_window = ctx_sampling->scratch; + string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, search_pos); size_t pos = 0; - if (string_is_found(window, elb.jumpword, pos)) { - LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(window).c_str(), search_pos + pos); + if (string_is_found(search_window, elb.jumpword, pos)) { + LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(search_window).c_str(), search_pos + pos); ctx_sampling->elb_idx = elb.jump_idx; search_pos += pos + elb.jumpword.length(); - } else if (string_is_found(window, elb.exitword, pos)) { - LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(window).c_str(), search_pos + pos); + } else if (string_is_found(search_window, elb.exitword, pos)) { + LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(search_window).c_str(), search_pos + pos); ctx_sampling->elb_idx++; search_pos += pos + elb.exitword.length(); } else { - search_pos += std::max(0, int32_t(window.length()) - elb.search_word_len); + search_pos += std::max(0, SSIZE(search_window) - elb.search_word_len); return; } @@ -1022,18 +1049,25 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); - const auto& rewinded_text = ctx_sampling->rewinded_text; - const auto& decoded_text = ctx_sampling->decoded_text; + // const auto& rewinded_text = ctx_sampling->rewinded_text; + + const int32_t net_text_len = SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(ctx_sampling->rewinded_text); + auto& elb = ctx_sampling->elb_states[idx]; + auto& window = ctx_sampling->scratch; for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { if (!entry.biases.empty()) { // no need to rewind elb continue; } + const auto& keywords = entry.phrases; + + const int32_t min_window_pos = net_text_len - entry.max_keyword_len + 1; + string_assign_append(window, ctx_sampling->generated_text, ctx_sampling->playing_text, min_window_pos); - for (size_t j = 0; j < entry.phrases.size(); ++j) { - const auto& phrase = entry.phrases[j]; - if (phrase.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { + for (size_t j = 0; j < keywords.size(); ++j) { + const auto& keyword = keywords[j]; + if (keyword.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { elb_add(ctx_sampling->params, entry); entry.addflags[j] = true; continue; @@ -1041,10 +1075,11 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { // triggered within rewinded text? size_t count = 0; - auto pos = decoded_text.find(phrase, std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - SSIZE(phrase) + 1)); + auto pos = window.find(keyword, entry.max_keyword_len - SSIZE(keyword) + std::max(0, elb.init_pos - min_window_pos)); + // auto pos = decoded_text.find(phrase, std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - SSIZE(keyword) + 1)); while (pos != std::string::npos) { ++count; - pos = decoded_text.find(phrase, pos + 1); + pos = window.find(keyword, pos + 1); } if (count % 2 == 1) { (entry.addflags[j] ? elb_sub : elb_add)(ctx_sampling->params, entry); @@ -1052,7 +1087,7 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } } } - ctx_sampling->elb_search_pos = std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - elb.search_word_len + 1); + ctx_sampling->elb_search_pos = std::max(elb.init_pos, net_text_len - elb.search_word_len + 1); LLAMA_LOG_DEBUG("%s[%d]:\elb_search_pos = %d, initpos %d, dtlen %d, rtlen %d, swlen %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos, elb.init_pos, SSIZE(decoded_text), SSIZE(rewinded_text), elb.search_word_len); diff --git a/common/sampling.h b/common/sampling.h index 9d2ab4dd5f..61bd662dfc 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -179,21 +179,23 @@ typedef struct common_params_sampling { // expiring logit bias (elb) + expiring sparam bias (epb) struct elb_param { struct elb_entry { - std::vector search_posi; // epb: starting search positions for phrases - std::vector addsubs; // epb: bias for sparams - std::vector addflags; // epb: true if added - std::vector phrases; // elb: exitwords OR epb: keywords - std::vector biases; // for each phrase, nth bias for nth token, extrapolate - int32_t duration; // bias duration, unless exitword matches - bool is_range; // has lower and upper biases + int32_t max_keyword_len; + std::vector search_posi; // epb: starting search positions for phrases + std::vector addsubs; // epb: bias for sparams + std::vector addflags; // epb: true if added + std::vector phrases; // exitwords for elb (below) OR keywords for epb (above) + std::vector biases; // elb: for each phrase, nth bias for nth token, extrapolate + int32_t duration; // elb: bias duration, unless exitword matches + bool is_range; // elb: has lower and upper biases bool operator == (const struct elb_entry& other) const { return (is_range == other.is_range) && (duration == other.duration) - && (biases == other.biases) - && (phrases == other.phrases) + && (max_keyword_len == other.max_keyword_len) && (addflags == other.addflags) + && (biases == other.biases) && (addsubs == other.addsubs) - && (search_posi == other.search_posi); + && (search_posi == other.search_posi) + && (phrases == other.phrases); } }; std::vector entries; @@ -217,6 +219,8 @@ struct common_sampler { bool is_decoding; // set true near the end of pp + std::string scratch; + // mirostat sampler state float mirostat_mu; @@ -238,7 +242,9 @@ struct common_sampler { int32_t n_rewind; std::string rewinded_text; - std::string decoded_text; + + std::string_view generated_text; + std::string playing_text; // token_buffer + draft llama_token_data_array cur_p; // current candidates @@ -266,7 +272,7 @@ struct common_sampler { int32_t init_pos; }; std::vector elb_states; - int32_t elb_idx; // for elb_states + int32_t elb_idx; // for elb_states and elb_params int32_t elb_search_pos; // for exitwords }; diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index f8338f29a6..527d196dc4 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -1769,7 +1769,6 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) common_sampler_free(slot.ctx_sampling); } slot.ctx_sampling = common_sampler_init(model, slot.sparams); - slot.ctx_sampling->decoded_text.reserve(4 * slot.n_ctx); } catch (std::exception & e) { std::string err_msg = std::string("Failed to initialize samplers: ") + e.what(); @@ -3990,6 +3989,12 @@ void server_context::speculative_decoding_accept() { apply_server_biases(slot); + slot.ctx_sampling->generated_text = slot.generated_text; + slot.ctx_sampling->playing_text.clear(); + for (const auto& token: slot.token_buffer) { + slot.ctx_sampling->playing_text.append(token.text_to_send); + } + // the accepted tokens from the speculation std::vector ids; try { @@ -4500,6 +4505,12 @@ void server_context::process_batch_tokens(int32_t & n_batch) { apply_server_biases(slot); + slot.ctx_sampling->generated_text = slot.generated_text; + slot.ctx_sampling->playing_text.clear(); + for (const auto& token: slot.token_buffer) { + slot.ctx_sampling->playing_text.append(token.text_to_send); + } + llama_token id; try { id = common_sampler_sample(slot.ctx_sampling, ctx, tok_idx); From 86a965bad6fb8395cd9dc6e7a33a1ed8ba393023 Mon Sep 17 00:00:00 2001 From: banana split Date: Tue, 16 Jun 2026 00:45:54 -0400 Subject: [PATCH 11/18] cleanup --- common/sampling.cpp | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 5891a6de69..b10d56509c 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -915,26 +915,16 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float continue; // next entry } - // search_window.clear(); - // margin = SSIZE(ctx_sampling->generated_text) - entry.search_posi[j]; - // if (margin > 0) { - // search_window.append(ctx_sampling->generated_text, entry.search_posi[j], -1).append(ctx_sampling->playing_text); - // } else { - // search_window.append(ctx_sampling->playing_text, -margin); - // } string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, entry.search_posi[j]); size_t count = 0; auto pos = search_window.find(phrase, entry.search_posi[j]); - // auto pos = ctx_sampling->decoded_text.find(phrase, entry.search_posi[j]); while (pos != std::string::npos) { LLAMA_LOG_DEBUG("%s: found %s @ %zu\n", __func__, phrase.c_str(), pos); ++count; pos = search_window.find(phrase, pos + phrase.length()); - // pos = ctx_sampling->decoded_text.find(phrase, pos + phrase.length()); } entry.search_posi[j] = SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(phrase) + 1; - // entry.search_posi[j] = std::max(0, int32_t(ctx_sampling->decoded_text.length()) - int32_t(phrase.length()) + 1); if (count % 2 == 1) { // even = no match or cancelled LLAMA_LOG_DEBUG("%s: before\n", __func__); @@ -963,15 +953,6 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru return; } - // std::string window; - // int32_t margin = SSIZE(ctx_sampling->generated_text) - ctx_sampling->elb_search_pos; - // window.reserve(margin + SSIZE(ctx_sampling->playing_text)); - // if (margin > 0) { - // window.append(ctx_sampling->generated_text, ctx_sampling->elb_search_pos, -1).append(ctx_sampling->playing_text); - // } else { - // window.append(ctx_sampling->playing_text, -margin); - // } - auto& search_pos = ctx_sampling->elb_search_pos; auto& search_window = ctx_sampling->scratch; string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, search_pos); @@ -1049,8 +1030,6 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); - // const auto& rewinded_text = ctx_sampling->rewinded_text; - const int32_t net_text_len = SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(ctx_sampling->rewinded_text); auto& elb = ctx_sampling->elb_states[idx]; @@ -1076,7 +1055,6 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { // triggered within rewinded text? size_t count = 0; auto pos = window.find(keyword, entry.max_keyword_len - SSIZE(keyword) + std::max(0, elb.init_pos - min_window_pos)); - // auto pos = decoded_text.find(phrase, std::max(elb.init_pos, SSIZE(decoded_text) - SSIZE(rewinded_text) - SSIZE(keyword) + 1)); while (pos != std::string::npos) { ++count; pos = window.find(keyword, pos + 1); @@ -1089,11 +1067,6 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } ctx_sampling->elb_search_pos = std::max(elb.init_pos, net_text_len - elb.search_word_len + 1); - LLAMA_LOG_DEBUG("%s[%d]:\elb_search_pos = %d, initpos %d, dtlen %d, rtlen %d, swlen %d\n", __func__, __LINE__, - ctx_sampling->elb_search_pos, elb.init_pos, SSIZE(decoded_text), SSIZE(rewinded_text), elb.search_word_len); - - LLAMA_LOG_DEBUG("%s[%d]: %s\n", __func__, __LINE__, decoded_text.c_str()); - // rewind sparam bias for (int32_t j = idx + 1; j < ctx_sampling->elb_states.size(); ++j) { for (auto& entry: ctx_sampling->params.elb_params[j].entries) { From 47f14d672c36f32a0f56d987d1a234fd7048ee9f Mon Sep 17 00:00:00 2001 From: banana split Date: Tue, 16 Jun 2026 22:38:09 -0400 Subject: [PATCH 12/18] fix: correct expiring logit bias search logic and server prompt ingestion This commit addresses several bugs in the sampling and server context: * Fixes a duplicate argument bug calculating search window length in ELB application. * Properly cleans up expiring logit bias states during a rewind. * Prevents the server from setting `is_decoding = true` before the prompt is fully ingested. * Minor debug logging fixes. --- common/common.cpp | 2 +- common/sampling.cpp | 63 +++++++++++++++++++++--------- examples/server/server-context.cpp | 2 +- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index c356c745b0..a6c00b1c1e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -5462,7 +5462,7 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp LLAMA_LOG_DEBUG("%s: line %zu: keyword = \"%s\"\n", __func__, i, keyword.c_str()); max_keyword_len = std::max(SSIZE(keyword), max_keyword_len); } - LLAMA_LOG_DEBUG("%s: line %zu: max_keyword_len = %zu\n", __func__, i, max_keyword_len); + LLAMA_LOG_DEBUG("%s: line %zu: max_keyword_len = %d\n", __func__, i, max_keyword_len); } common_params_sampling::elb_param::elb_entry entry = { diff --git a/common/sampling.cpp b/common/sampling.cpp index b10d56509c..b86efeab11 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -846,10 +846,12 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float }; auto& search_window = ctx_sampling->scratch; + // printf("%s[%d]\n", __func__, __LINE__); string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, - std::max(0, SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->generated_text) - elb.max_cond_len)); + std::max(0, SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - elb.max_cond_len)); + // printf("%s[%d]\n", __func__, __LINE__); if (!search_window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.other_tokens); @@ -914,8 +916,9 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } continue; // next entry } - + // printf("%s[%d]\n", __func__, __LINE__); string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, entry.search_posi[j]); + // printf("%s[%d]\n", __func__, __LINE__); size_t count = 0; auto pos = search_window.find(phrase, entry.search_posi[j]); @@ -948,14 +951,15 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru } auto& elb = ctx_sampling->elb_states[idx]; - // LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, %s\n", __func__, __LINE__, idx, elb.countup, string_unescape(common_token_to_piece(ctx_main, ctx_sampling->prev.back(), true)).c_str()); if ((elb.delay > ++(elb.countup)) || (elb.search_word_len == 0)) { return; } auto& search_pos = ctx_sampling->elb_search_pos; auto& search_window = ctx_sampling->scratch; + // printf("%s[%d]: %s, %s, %d\n", __func__, __LINE__, std::string(ctx_sampling->generated_text).c_str(), ctx_sampling->playing_text.c_str(), search_pos); string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, search_pos); + // printf("%s[%d]\n", __func__, __LINE__); size_t pos = 0; if (string_is_found(search_window, elb.jumpword, pos)) { @@ -1012,6 +1016,23 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { // LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind); + if (n_rewind > ctx_sampling->elb_states[idx].countup) { + // leaving current state so cleanup + for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { + for (const auto addflag: entry.addflags) { + if (addflag) { + LLAMA_LOG_DEBUG("%s: before\n", __func__); + elb_print(ctx_sampling->params, entry); + + elb_sub(ctx_sampling->params, entry); + + LLAMA_LOG_DEBUG("%s: after\n", __func__); + elb_print(ctx_sampling->params, entry); + } + } + } + } + // consume n_rewind while (n_rewind > 0) { while (ctx_sampling->elb_states[idx].countup == 0) { @@ -1036,13 +1057,15 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { auto& window = ctx_sampling->scratch; for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { if (!entry.biases.empty()) { - // no need to rewind elb + // no need to rewind logit biases continue; } const auto& keywords = entry.phrases; const int32_t min_window_pos = net_text_len - entry.max_keyword_len + 1; + // printf("%s[%d]\n", __func__, __LINE__); string_assign_append(window, ctx_sampling->generated_text, ctx_sampling->playing_text, min_window_pos); + // printf("%s[%d]\n", __func__, __LINE__); for (size_t j = 0; j < keywords.size(); ++j) { const auto& keyword = keywords[j]; @@ -1067,22 +1090,26 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } ctx_sampling->elb_search_pos = std::max(elb.init_pos, net_text_len - elb.search_word_len + 1); - // rewind sparam bias - for (int32_t j = idx + 1; j < ctx_sampling->elb_states.size(); ++j) { - for (auto& entry: ctx_sampling->params.elb_params[j].entries) { - for (const auto addflag: entry.addflags) { - if (addflag) { - LLAMA_LOG_DEBUG("%s: before\n", __func__); - elb_print(ctx_sampling->params, entry); + for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { + entry.search_posi.assign(entry.search_posi.size(), elb.init_pos ); + } - elb_sub(ctx_sampling->params, entry); + // // rewind sparam bias + // for (int32_t j = idx + 1; j < ctx_sampling->elb_states.size(); ++j) { + // for (auto& entry: ctx_sampling->params.elb_params[j].entries) { + // for (const auto addflag: entry.addflags) { + // if (addflag) { + // LLAMA_LOG_DEBUG("%s: before\n", __func__); + // elb_print(ctx_sampling->params, entry); - LLAMA_LOG_DEBUG("%s: after\n", __func__); - elb_print(ctx_sampling->params, entry); - } - } - } - } + // elb_sub(ctx_sampling->params, entry); + + // LLAMA_LOG_DEBUG("%s: after\n", __func__); + // elb_print(ctx_sampling->params, entry); + // } + // } + // } + // } } diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 527d196dc4..e7d2687e21 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -3917,13 +3917,13 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t GGML_ASSERT(batch.n_tokens > 0); GGML_ASSERT((size_t)slot.n_prompt_tokens == slot.prompt_tokens.size()); common_sampler_reset(slot.ctx_sampling); - slot.ctx_sampling->is_decoding = true; for (int i = 0; i < slot.n_prompt_tokens; ++i) { llama_token id = slot.prompt_tokens[i]; if (id != LLAMA_TOKEN_NULL) { common_sampler_accept(slot.ctx_sampling, ctx, id, false); } } + slot.ctx_sampling->is_decoding = true; // extract the logits only for the last token batch.logits[batch.n_tokens - 1] = true; From 507927e0b7722f333f26307bcb3eb6cf4d788105 Mon Sep 17 00:00:00 2001 From: banana split Date: Tue, 16 Jun 2026 23:29:32 -0400 Subject: [PATCH 13/18] cleanup --- common/sampling.cpp | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index b86efeab11..d572a63d2a 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -957,9 +957,7 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru auto& search_pos = ctx_sampling->elb_search_pos; auto& search_window = ctx_sampling->scratch; - // printf("%s[%d]: %s, %s, %d\n", __func__, __LINE__, std::string(ctx_sampling->generated_text).c_str(), ctx_sampling->playing_text.c_str(), search_pos); string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, search_pos); - // printf("%s[%d]\n", __func__, __LINE__); size_t pos = 0; if (string_is_found(search_window, elb.jumpword, pos)) { @@ -1063,19 +1061,18 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { const auto& keywords = entry.phrases; const int32_t min_window_pos = net_text_len - entry.max_keyword_len + 1; - // printf("%s[%d]\n", __func__, __LINE__); string_assign_append(window, ctx_sampling->generated_text, ctx_sampling->playing_text, min_window_pos); - // printf("%s[%d]\n", __func__, __LINE__); for (size_t j = 0; j < keywords.size(); ++j) { const auto& keyword = keywords[j]; if (keyword.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) { + // redo duration-based bias elb_add(ctx_sampling->params, entry); entry.addflags[j] = true; continue; } - // triggered within rewinded text? + // keyword match within rewinded text? size_t count = 0; auto pos = window.find(keyword, entry.max_keyword_len - SSIZE(keyword) + std::max(0, elb.init_pos - min_window_pos)); while (pos != std::string::npos) { @@ -1088,28 +1085,12 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) { } } } - ctx_sampling->elb_search_pos = std::max(elb.init_pos, net_text_len - elb.search_word_len + 1); + // reinitialize + ctx_sampling->elb_search_pos = std::max(elb.init_pos, net_text_len - elb.search_word_len + 1); for (auto& entry: ctx_sampling->params.elb_params[idx].entries) { - entry.search_posi.assign(entry.search_posi.size(), elb.init_pos ); + entry.search_posi.assign(entry.search_posi.size(), elb.init_pos); } - - // // rewind sparam bias - // for (int32_t j = idx + 1; j < ctx_sampling->elb_states.size(); ++j) { - // for (auto& entry: ctx_sampling->params.elb_params[j].entries) { - // for (const auto addflag: entry.addflags) { - // if (addflag) { - // LLAMA_LOG_DEBUG("%s: before\n", __func__); - // elb_print(ctx_sampling->params, entry); - - // elb_sub(ctx_sampling->params, entry); - - // LLAMA_LOG_DEBUG("%s: after\n", __func__); - // elb_print(ctx_sampling->params, entry); - // } - // } - // } - // } } From 7a7617d4ddc4c40fa3178cc622ad680d23678671 Mon Sep 17 00:00:00 2001 From: banana split Date: Thu, 18 Jun 2026 01:15:50 -0400 Subject: [PATCH 14/18] compatibility patch speculative decoding and string ban --- common/sampling.cpp | 5 +---- common/sampling.h | 2 +- examples/server/server-context.cpp | 21 +++++++++++++++++---- examples/server/server-context.h | 2 ++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index d572a63d2a..088b42d10b 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -846,12 +846,10 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float }; auto& search_window = ctx_sampling->scratch; - // printf("%s[%d]\n", __func__, __LINE__); string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, std::max(0, SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - elb.max_cond_len)); - // printf("%s[%d]\n", __func__, __LINE__); if (!search_window.empty() && !elb.other_tokens.empty() && (elb.other_tokens.front().duration > elb.countup)) { const auto ifi = index_first_inactive(elb.other_tokens); @@ -916,9 +914,8 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float } continue; // next entry } - // printf("%s[%d]\n", __func__, __LINE__); + string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, entry.search_posi[j]); - // printf("%s[%d]\n", __func__, __LINE__); size_t count = 0; auto pos = search_window.find(phrase, entry.search_posi[j]); diff --git a/common/sampling.h b/common/sampling.h index 61bd662dfc..41177ed844 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -11,7 +11,7 @@ #define A_DOT_B(a, b) a.b -#define SSIZE(container) int32_t(container.size()) +#define SSIZE(container) static_cast(container.size()) // sampler types enum class llama_sampler_type : char { diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index e7d2687e21..129c79a813 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -1838,7 +1838,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) } } - slot.ctx_sampling->elb_states.push_back({ { }, { }, exitword, 0, 0, 0, "", 0, int32_t(exitword.length()), 0 }); + slot.ctx_sampling->elb_states.push_back({ { }, { }, exitword, 0, 0, 0, "", 0, SSIZE(exitword), 0 }); auto& first_tokens = slot.ctx_sampling->elb_states.back().first_tokens; auto& other_tokens = slot.ctx_sampling->elb_states.back().other_tokens; @@ -4081,6 +4081,20 @@ void server_context::speculative_decoding_accept() { } } + if (slot.ctx_sampling->n_rewind > 0) { + // consume out-of-context tokens + auto banned_n = slot.banned_n; + slot.banned_n = 0; + result.prob = 0.0f; + for (++i; i < ids.size(); ++i) { + result.tok = ids[i]; + result.text_to_send = common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)); + slot.token_buffer.push_back(result); + rewind_context(slot, slot.n_past); + } + slot.banned_n = banned_n; + } + common_sampler_review(slot.ctx_sampling, slot.token_buffer.size(), slot.rewind_status); update_allowlist_state(slot); @@ -4225,15 +4239,14 @@ inline int32_t check_ban_phrase(server_slot& slot) { return -1; } -inline void rewind_context(server_slot& slot, int32_t ban_pos) { +void server_context::rewind_context(server_slot& slot, int32_t ban_pos) { slot.rewind_count++; int32_t buffer_start_pos = slot.n_past - (int32_t)slot.token_buffer.size() + 1; int32_t n_keep_buffer = ban_pos - buffer_start_pos; if (n_keep_buffer < 0) n_keep_buffer = 0; - slot.ctx_sampling->n_rewind = slot.token_buffer.size() - n_keep_buffer; - slot.ctx_sampling->rewinded_text.reserve(4 * slot.ctx_sampling->n_rewind); + slot.ctx_sampling->n_rewind += slot.token_buffer.size() - n_keep_buffer; LLAMA_LOG_DEBUG("%s: rewinding %d tokens\n", __func__, slot.ctx_sampling->n_rewind); for (int32_t j = n_keep_buffer; j < slot.token_buffer.size(); ++j) { slot.ctx_sampling->rewinded_text.append(slot.token_buffer[j].text_to_send); diff --git a/examples/server/server-context.h b/examples/server/server-context.h index 44538c641e..6099685323 100644 --- a/examples/server/server-context.h +++ b/examples/server/server-context.h @@ -373,6 +373,8 @@ struct server_context { void send_token_results(completion_token_outputs& results, server_slot& slot, int32_t n = 0); + void rewind_context(server_slot& slot, int32_t ban_pos); + void buffer_and_check_string_ban(server_slot& slot, completion_token_output& result); void update_allowlist_state(server_slot& slot); From 7d8ddb802c90bc23c05f7cc54583653d9ec1d0e2 Mon Sep 17 00:00:00 2001 From: banana split Date: Fri, 19 Jun 2026 18:48:25 -0400 Subject: [PATCH 15/18] minor --- examples/server/server-context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 129c79a813..aa039beb04 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -1898,7 +1898,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) } other_tokens.push_back({ ids[j], biases[j], size_t(duration + m * j), cond }); } - max_cond_len = std::max(int32_t(cond.length()), max_cond_len); + max_cond_len = std::max(SSIZE(cond), max_cond_len); } } } From 0b71102f40dd2115b8cbe8078ed5cb389a868ebc Mon Sep 17 00:00:00 2001 From: banana split Date: Sat, 20 Jun 2026 00:51:40 -0400 Subject: [PATCH 16/18] fix keyword search --- common/sampling.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 088b42d10b..9c74052f58 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -918,7 +918,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float string_assign_append(search_window, ctx_sampling->generated_text, ctx_sampling->playing_text, entry.search_posi[j]); size_t count = 0; - auto pos = search_window.find(phrase, entry.search_posi[j]); + auto pos = search_window.find(phrase); while (pos != std::string::npos) { LLAMA_LOG_DEBUG("%s: found %s @ %zu\n", __func__, phrase.c_str(), pos); ++count; From a36f23f6e7ba21dea1d119f8c012b328621df280 Mon Sep 17 00:00:00 2001 From: banana split Date: Sun, 21 Jun 2026 01:29:25 -0400 Subject: [PATCH 17/18] move search position to include next token --- common/sampling.cpp | 2 +- examples/server/server-context.cpp | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index 9c74052f58..84dc84e2da 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -966,7 +966,7 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru ctx_sampling->elb_idx++; search_pos += pos + elb.exitword.length(); } else { - search_pos += std::max(0, SSIZE(search_window) - elb.search_word_len); + search_pos += std::max(0, SSIZE(search_window) - elb.search_word_len + 1); return; } diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 4c82bf7b0c..9e64335e8d 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -4136,11 +4136,12 @@ void server_context::speculative_decoding_accept() { // consume out-of-context tokens auto banned_n = slot.banned_n; slot.banned_n = 0; - result.prob = 0.0f; for (++i; i < ids.size(); ++i) { - result.tok = ids[i]; - result.text_to_send = common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)); - slot.token_buffer.push_back(result); + slot.token_buffer.push_back({ + ids[i], + common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)), + 0.0f, + { } }); rewind_context(slot, slot.n_past); } slot.banned_n = banned_n; @@ -4298,7 +4299,7 @@ void server_context::rewind_context(server_slot& slot, int32_t ban_pos) { if (n_keep_buffer < 0) n_keep_buffer = 0; slot.ctx_sampling->n_rewind += slot.token_buffer.size() - n_keep_buffer; - LLAMA_LOG_DEBUG("%s: rewinding %d tokens\n", __func__, slot.ctx_sampling->n_rewind); + LLAMA_LOG_DEBUG("%s[%d]: n_rewind = %d\n", __func__, __LINE__, slot.ctx_sampling->n_rewind); for (int32_t j = n_keep_buffer; j < slot.token_buffer.size(); ++j) { slot.ctx_sampling->rewinded_text.append(slot.token_buffer[j].text_to_send); } From 4e82d8341bdb9ea53361f6e11a74a82681e56f9e Mon Sep 17 00:00:00 2001 From: banana split Date: Wed, 24 Jun 2026 23:04:14 -0400 Subject: [PATCH 18/18] fix underflow --- common/common.cpp | 2 +- common/sampling.cpp | 2 +- common/sampling.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2ce90d868b..a50b814310 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -5483,7 +5483,7 @@ void argparse_expiring_logit_bias(const std::string& content, common_params_samp common_params_sampling::elb_param::elb_entry entry = { max_keyword_len, - std::vector(n_phrase, 0), + std::vector(n_phrase, 0), std::move(addsubs), std::vector(n_phrase, false), std::move(phrases), diff --git a/common/sampling.cpp b/common/sampling.cpp index 84dc84e2da..8e7c9f456c 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -924,7 +924,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float ++count; pos = search_window.find(phrase, pos + phrase.length()); } - entry.search_posi[j] = SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(phrase) + 1; + entry.search_posi[j] = std::max(0, SSIZE(ctx_sampling->generated_text) + SSIZE(ctx_sampling->playing_text) - SSIZE(phrase) + 1); if (count % 2 == 1) { // even = no match or cancelled LLAMA_LOG_DEBUG("%s: before\n", __func__); diff --git a/common/sampling.h b/common/sampling.h index 41177ed844..7f52e5873e 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -180,7 +180,7 @@ typedef struct common_params_sampling { struct elb_param { struct elb_entry { int32_t max_keyword_len; - std::vector search_posi; // epb: starting search positions for phrases + std::vector search_posi; // epb: starting search positions for phrases std::vector addsubs; // epb: bias for sparams std::vector addflags; // epb: true if added std::vector phrases; // exitwords for elb (below) OR keywords for epb (above)