Skip to content

Commit c7d9568

Browse files
committed
Add -igm / --ignore-recurrent-model flag to disable recurrent model checkpoint behavior
When a recurrent model (e.g. Qwen3.5-MoE) is used with the server, checkpoints, banned strings disabling, and ctx_shift disabling are automatically applied (from commit 'server: enable checkpoint for recurrent models (ikawrakow#1310)'). The new -igm / --ignore-recurrent-model flag skips all this, making recurrent models behave as non-recurrent. - Added ignore_recurrent_model field to gpt_params (default: false) - Added CLI parsing for -igm / --ignore-recurrent-model - Wrapped the recurrent model block in server-context.cpp with the flag check
1 parent 659526c commit c7d9568

3 files changed

Lines changed: 24 additions & 16 deletions

File tree

common/common.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2945,6 +2945,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
29452945
params.ctx_checkpoints_tolerance = std::stoi(argv[i]);
29462946
return true;
29472947
}
2948+
if (arg == "-igm" || arg == "--ignore-recurrent-model") {
2949+
params.ignore_recurrent_model = true;
2950+
return true;
2951+
}
29482952
if (arg == "-cram" || arg == "--cache-ram") {
29492953
CHECK_ARG
29502954
params.cache_ram_mib = std::stoi(argv[i]);
@@ -3162,6 +3166,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
31623166
options.push_back({ "*", "-ctx-ck, --ctx-checkpoints N", "max number of context checkpoints to create per slot (default: %d)",params.ctx_checkpoints_n});
31633167
options.push_back({ "*", "-ctx-ck-i, --ctx-checkpoints-interval N", "minimum number of tokens between each context checkpoint. (default: %d, <=0 disable)",params.ctx_checkpoints_interval});
31643168
options.push_back({ "*", "-ctx-ck-t, --ctx-checkpoints-tolerance N", "the number of tokens before the full prompt to create the checkpoint. (default: %d, <=0 disable)",params.ctx_checkpoints_tolerance});
3169+
options.push_back({ "*", "-igm, --ignore-recurrent-model", "disable recurrent model checkpoint behavior, treat as non-recurrent"});
31653170
options.push_back({ "*", "-cram, --cache-ram N", "set the maximum cache size in MiB (default: %d, -1 - no limit, 0 - disable)",params.cache_ram_mib });
31663171
options.push_back({ "*", "-crs, --cache-ram-similarity N", "max of similarity of prompt tokens to cache tokens that triggers prompt cache (default: %.2f).",params.cache_ram_similarity });
31673172
options.push_back({ "*", "-cram-n-min --cache-ram-n-min N", "minimum number of the cached tokens that triggers prompt cache (default: %d).", params.cache_ram_n_min });

common/common.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,7 @@ struct gpt_params {
562562
float slot_prompt_similarity = 0.1f;
563563

564564
bool do_checkpoint = false; // do checkpoint for recurrent models only
565+
bool ignore_recurrent_model = false; // treat recurrent models as non-recurrent (disable checkpoints)
565566
int32_t ctx_checkpoints_n = 32; // max number of context checkpoints per slot
566567
int32_t ctx_checkpoints_interval = n_batch * 2; // minimum number of tokens between each context checkpoints
567568
int32_t ctx_checkpoints_tolerance = 5; // the number of tokens before the full prompt to create the checkpoint

examples/server/server-context.cpp

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1999,22 +1999,24 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
19991999
slot.allow_ruless_prev = slot.allow_ruless;
20002000

20012001
if (llama_model_has_recurrent(llama_get_model(slot.ctx))) {
2002-
params_base.can_ban_phrases = false;
2003-
bool do_checkpoint = params_base.ctx_checkpoints_n > 0;
2004-
// make checkpoints only for completion tasks
2005-
do_checkpoint = do_checkpoint && task.type == SERVER_TASK_TYPE_COMPLETION;
2006-
// make a checkpoint of the parts of the memory that cannot be rolled back.
2007-
// checkpoints are created only if:
2008-
// - the model architecture is marked as recurrent or hybrid
2009-
//
2010-
// TODO: try to make this conditional on the context or the memory module, instead of the model type
2011-
params_base.do_checkpoint = do_checkpoint;
2012-
if (slot.n_buffer != 0) {
2013-
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
2014-
}
2015-
if (params_base.ctx_shift) {
2016-
params_base.ctx_shift = false;
2017-
LOG_WARNING("%s\n", "ctx_shift is not supported by recurrent model, it will be disabled");
2002+
if (!params_base.ignore_recurrent_model) {
2003+
params_base.can_ban_phrases = false;
2004+
bool do_checkpoint = params_base.ctx_checkpoints_n > 0;
2005+
// make checkpoints only for completion tasks
2006+
do_checkpoint = do_checkpoint && task.type == SERVER_TASK_TYPE_COMPLETION;
2007+
// make a checkpoint of the parts of the memory that cannot be rolled back.
2008+
// checkpoints are created only if:
2009+
// - the model architecture is marked as recurrent or hybrid
2010+
//
2011+
// TODO: try to make this conditional on the context or the memory module, instead of the model type
2012+
params_base.do_checkpoint = do_checkpoint;
2013+
if (slot.n_buffer != 0) {
2014+
LLAMA_LOG_WARN("banned strings is not supported by recurrent model, it will be disabled.\n");
2015+
}
2016+
if (params_base.ctx_shift) {
2017+
params_base.ctx_shift = false;
2018+
LOG_WARNING("%s\n", "ctx_shift is not supported by recurrent model, it will be disabled");
2019+
}
20182020
}
20192021
}
20202022
if (llama_model_is_split_mode_tensor_parallel(llama_get_model(slot.ctx))) {

0 commit comments

Comments
 (0)