Skip to content

server: adaptive low-yield MTP speculation fallback #22931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
leon7609 wants to merge 1 commit into from
Closed

server: adaptive low-yield MTP speculation fallback #22931

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions tools/server/server-context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ struct server_slot {
std::vector<int32_t> spec_i_batch;
server_prompt_checkpoint spec_ckpt;
common_speculative_ptr spec;
bool spec_runtime_disabled = false;
int32_t spec_verify_batches = 0;
int32_t spec_verify_draft_total = 0;
int32_t spec_verify_draft_accepted = 0;

// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
Expand Down Expand Up @@ -218,6 +222,10 @@ struct server_slot {
// clear speculative decoding stats
n_draft_total = 0;
n_draft_accepted = 0;
spec_runtime_disabled = false;
spec_verify_batches = 0;
spec_verify_draft_total = 0;
spec_verify_draft_accepted = 0;

task_prev = std::move(task);
task.reset();
Expand Down Expand Up @@ -310,6 +318,10 @@ struct server_slot {
return 0;
}

if (spec_runtime_disabled && spec_draft.empty()) {
return 0;
}

const int n_draft_min = common_speculative_n_min(spec.get(), task->params.speculative);

// determine the max draft that fits the current slot state
Expand Down Expand Up @@ -2999,6 +3011,24 @@ struct server_context_impl {

GGML_ASSERT(accepted.size() >= 1);

const int32_t n_draft_accepted_actual = std::max<int32_t>(0,
std::min<int32_t>((int32_t) n_draft, (int32_t) accepted.size() - 1));
slot.spec_verify_batches += 1;
slot.spec_verify_draft_total += (int32_t) n_draft;
slot.spec_verify_draft_accepted += n_draft_accepted_actual;
if (slot.task->params.speculative.type == COMMON_SPECULATIVE_TYPE_MTP &&
!slot.spec_runtime_disabled &&
slot.spec_verify_batches >= 2 &&
slot.spec_verify_draft_total >= 16 &&
4 * slot.spec_verify_draft_accepted < 3 * slot.spec_verify_draft_total) {
slot.spec_runtime_disabled = true;
SLT_WRN(slot,
"disabling MTP speculation for this request: actual draft acceptance %d/%d after %d verify batches\n",
slot.spec_verify_draft_accepted,
slot.spec_verify_draft_total,
slot.spec_verify_batches);
}

// check for partial draft acceptance
if (accepted.size() < slot.spec_draft.size() + 1) {
if (use_ckpt) {
Expand Down