diff --git a/src/llama.cpp b/src/llama.cpp
index f73a2bda01..5e04b7764d 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -966,9 +966,9 @@ static bool llama_kv_cache_init(
                 k->extra = (void *)&split_k_l.ggml;
                 v->extra = (void *)&split_v_l.ggml;
             }
+            cache.k_l.push_back(k);
+            cache.v_l.push_back(v);
         }
-        cache.k_l.push_back(k);
-        cache.v_l.push_back(v);
     }
     if (is_mla_attn && cparams.mla_attn && n_mla < n_layer && n_mla > 0) {
         LLAMA_LOG_ERROR("%s: unexpected situation with %d out of %d layers having MLA enabled\n", __func__, n_mla, int(n_layer));