Skip to content

Commit c51c47f

Browse files
Green-Skyspiritbuun
authored andcommitted
kv-cache : extend cache quantization checks (#21586)
to also check for enabled flash attention, instead of just auto.
1 parent 69618c4 commit c51c47f

1 file changed

Lines changed: 2 additions & 2 deletions

File tree

src/llama-context.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2959,7 +2959,7 @@ llama_context * llama_init_from_model(
29592959
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
29602960
}
29612961

2962-
if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_k)) {
2962+
if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {
29632963
const uint32_t blck_size = ggml_blck_size(params.type_k);
29642964
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
29652965
if (model->hparams.n_embd_head_k(il) % blck_size != 0) {
@@ -2970,7 +2970,7 @@ llama_context * llama_init_from_model(
29702970
}
29712971
}
29722972

2973-
if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO && ggml_is_quantized(params.type_v)) {
2973+
if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_v)) {
29742974
const uint32_t blck_size = ggml_blck_size(params.type_v);
29752975
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
29762976
if (model->hparams.n_embd_head_v(il) % blck_size != 0) {

0 commit comments

Comments
 (0)