Skip to content

Commit f1abc91

Browse files
committed
Add -igm / --ignore-recurrent-model flag to disable recurrent model checkpoint behavior
When a recurrent model (e.g. Qwen3.5-MoE) is used with the server, checkpoints, banned strings disabling, and ctx_shift disabling are automatically applied (from commit 'server: enable checkpoint for recurrent models (ikawrakow#1310)'). The new -igm / --ignore-recurrent-model flag skips all this, making recurrent models behave as non-recurrent. - Added ignore_recurrent_model field to gpt_params (default: false) - Added CLI parsing for -igm / --ignore-recurrent-model - Wrapped the recurrent model block in server-context.cpp with the flag check
1 parent eab0359 commit f1abc91

3 files changed

Lines changed: 24 additions & 16 deletions

File tree

common/common.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3054,6 +3054,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
30543054
params.ctx_checkpoints_tolerance = std::stoi(argv[i]);
30553055
return true;
30563056
}
3057+
if (arg == "-igm" || arg == "--ignore-recurrent-model") {
3058+
params.ignore_recurrent_model = true;
3059+
return true;
3060+
}
30573061
if (arg == "-cram" || arg == "--cache-ram") {
30583062
CHECK_ARG
30593063
params.cache_ram_mib = std::stoi(argv[i]);
@@ -3271,6 +3275,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
32713275
options.push_back({ "*", "-ctx-ck, --ctx-checkpoints N", "max number of context checkpoints to create per slot (default: %d)",params.ctx_checkpoints_n});
32723276
options.push_back({ "*", "-ctx-ck-i, --ctx-checkpoints-interval N", "minimum number of tokens between each context checkpoint. (default: %d, <=0 disable)",params.ctx_checkpoints_interval});
32733277
options.push_back({ "*", "-ctx-ck-t, --ctx-checkpoints-tolerance N", "the number of tokens before the full prompt to create the checkpoint. (default: %d, <=0 disable)",params.ctx_checkpoints_tolerance});
3278+
options.push_back({ "*", "-igm, --ignore-recurrent-model", "disable recurrent model checkpoint behavior, treat as non-recurrent"});
32743279
options.push_back({ "*", "-cram, --cache-ram N", "set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)",params.cache_ram_mib });
32753280
options.push_back({ "*", "-crs, --cache-ram-similarity N", "max of similarity of prompt tokens to cache tokens that triggers prompt cache (default: %.2f).",params.cache_ram_similarity });
32763281
options.push_back({ "*", "-cram-n-min --cache-ram-n-min N", "minimum number of the cached tokens that triggers prompt cache (default: %d).", params.cache_ram_n_min });

common/common.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,7 @@ struct gpt_params {
572572
float slot_prompt_similarity = 0.1f;
573573

574574
bool do_checkpoint = false; // do checkpoint for recurrent models only
575+
bool ignore_recurrent_model = false; // treat recurrent models as non-recurrent (disable checkpoints)
575576
int32_t ctx_checkpoints_n = 32; // max number of context checkpoints per slot
576577
int32_t ctx_checkpoints_interval = n_batch * 2; // minimum number of tokens between each context checkpoints
577578
int32_t ctx_checkpoints_tolerance = 5; // the number of tokens before the full prompt to create the checkpoint

examples/server/server-context.cpp

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1720,22 +1720,24 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
17201720
slot.allow_ruless_prev = slot.allow_ruless;
17211721

17221722
if (llama_model_has_recurrent(llama_get_model(slot.ctx))) {
1723-
params_base.can_ban_phrases = false;
1724-
bool do_checkpoint = params_base.ctx_checkpoints_n > 0;
1725-
// make checkpoints only for completion tasks
1726-
do_checkpoint = do_checkpoint && task.type == SERVER_TASK_TYPE_COMPLETION;
1727-
// make a checkpoint of the parts of the memory that cannot be rolled back.
1728-
// checkpoints are created only if:
1729-
// - the model architecture is marked as recurrent or hybrid
1730-
//
1731-
// TODO: try to make this conditional on the context or the memory module, instead of the model type
1732-
params_base.do_checkpoint = do_checkpoint;
1733-
if (slot.n_buffer != 0) {
1734-
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
1735-
}
1736-
if (params_base.ctx_shift) {
1737-
params_base.ctx_shift = false;
1738-
LOG_WARNING("%s\n", "ctx_shift is not supported by recurrent model, it will be disabled");
1723+
if (!params_base.ignore_recurrent_model) {
1724+
params_base.can_ban_phrases = false;
1725+
bool do_checkpoint = params_base.ctx_checkpoints_n > 0;
1726+
// make checkpoints only for completion tasks
1727+
do_checkpoint = do_checkpoint && task.type == SERVER_TASK_TYPE_COMPLETION;
1728+
// make a checkpoint of the parts of the memory that cannot be rolled back.
1729+
// checkpoints are created only if:
1730+
// - the model architecture is marked as recurrent or hybrid
1731+
//
1732+
// TODO: try to make this conditional on the context or the memory module, instead of the model type
1733+
params_base.do_checkpoint = do_checkpoint;
1734+
if (slot.n_buffer != 0) {
1735+
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
1736+
}
1737+
if (params_base.ctx_shift) {
1738+
params_base.ctx_shift = false;
1739+
LOG_WARNING("%s\n", "ctx_shift is not supported by recurrent model, it will be disabled");
1740+
}
17391741
}
17401742
}
17411743
if (llama_model_is_split_mode_tensor_parallel(llama_get_model(slot.ctx))) {

0 commit comments

Comments
 (0)