diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 187dfe8001..75ba23e9c2 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -594,6 +594,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8": # ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01 res = "command-r" + if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e": + # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0 + res = "cohere2_moe" if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" @@ -1561,7 +1564,12 @@ def set_vocab(self): special_vocab.add_to_gguf(self.gguf_writer) def set_gguf_parameters(self): + saved_intermediate_size = self.hparams.get("intermediate_size") + saved_num_experts_per_tok = self.hparams.pop("num_experts_per_tok") + self.hparams["intermediate_size"] = self.hparams["prefix_dense_intermediate_size"] super().set_gguf_parameters() + self.hparams["intermediate_size"] = saved_intermediate_size + self.hparams["num_experts_per_tok"] = saved_num_experts_per_tok hparams = self.hparams self.gguf_writer.add_vocab_size(hparams["vocab_size"]) @@ -3692,6 +3700,86 @@ def set_gguf_parameters(self): self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) +@Model.register("Cohere2MoeForCausalLM") +class Cohere2MoeModel(Model): + model_arch = gguf.MODEL_ARCH.COHERE2_MOE + + _experts: list[dict[str, Tensor]] | None = None + + def set_gguf_parameters(self): + saved_intermediate_size = self.hparams["intermediate_size"] + saved_num_experts_per_tok = self.hparams.pop("num_experts_per_tok") + self.hparams["intermediate_size"] = self.hparams["prefix_dense_intermediate_size"] + super().set_gguf_parameters() + self.hparams["intermediate_size"] = saved_intermediate_size + self.hparams["num_experts_per_tok"] = saved_num_experts_per_tok + hparams = self.hparams + + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_logit_scale(hparams.get("logit_scale", 1.0)) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([ + layer_type == "sliding_attention" + for layer_type in hparams["layer_types"] + ]) + self.gguf_writer.add_rope_dimension_count(hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"])) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + self.gguf_writer.add_expert_feed_forward_length(hparams["intermediate_size"]) + self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"]) + self.gguf_writer.add_expert_count(hparams["num_experts"]) + self.gguf_writer.add_expert_used_count(hparams["num_experts_per_tok"]) + self.gguf_writer.add_expert_weights_norm(bool(hparams.get("norm_topk_prob", False))) + + expert_selection_fn = hparams.get("expert_selection_fn", "softmax") + if expert_selection_fn != "sigmoid": + raise ValueError(f"Unsupported Cohere2-MoE expert_selection_fn={expert_selection_fn!r}") + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + + if hparams.get("num_shared_experts", 0) != 0: + raise ValueError("Cohere2-MoE shared experts are not supported in this GGUF converter yet") + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # Cohere2-MoE HF tensors already use the interleaved RoPE layout expected here. + + if ".mlp.experts." in name: + n_experts = self.hparams["num_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) < n_experts * 3: + return [] + + tensors: list[tuple[str, Tensor]] = [] + for src, dst in [ + ("gate_proj", "gate_proj"), + ("down_proj", "down_proj"), + ("up_proj", "up_proj"), + ]: + datas: list[Tensor] = [] + for xid in range(n_experts): + ename = f"model.layers.{bid}.mlp.experts.{xid}.{src}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + merged_name = f"model.layers.{bid}.mlp.experts.{dst}.weight" + tensors.append((self.map_tensor_name(merged_name), torch.stack(datas, dim=0))) + yield from tensors + return + + if name == "model.embed_tokens.weight": + yield self.map_tensor_name(name), data_torch + if self.tensor_names is None or "lm_head.weight" not in self.tensor_names: + yield self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".weight"), data_torch + return + + yield self.map_tensor_name(name), data_torch + + @Model.register("OlmoForCausalLM") @Model.register("OLMoForCausalLM") class OlmoModel(Model): diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 0c81201b84..ca965e3360 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -250,6 +250,7 @@ class MODEL_ARCH(IntEnum): MAMBA = auto() XVERSE = auto() COMMAND_R = auto() + COHERE2_MOE = auto() DBRX = auto() OLMO = auto() OPENELM = auto() @@ -419,6 +420,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.MAMBA: "mamba", MODEL_ARCH.XVERSE: "xverse", MODEL_ARCH.COMMAND_R: "command-r", + MODEL_ARCH.COHERE2_MOE: "cohere2_moe", MODEL_ARCH.DBRX: "dbrx", MODEL_ARCH.OLMO: "olmo", MODEL_ARCH.OPENELM: "openelm", @@ -1136,6 +1138,23 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_Q_NORM, ], + MODEL_ARCH.COHERE2_MOE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], MODEL_ARCH.DBRX: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index e1d5aaf47a..fb3ea08063 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -278,7 +278,13 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: elif chat_template_json.is_file(): with open(chat_template_json, encoding = 'utf-8') as f: chat_template_alt = json.load(f).get('chat_template') - chat_template = tokenizer_config.get('chat_template', chat_template_alt) + prefer_chat_template_alt = False + if chat_template_alt is not None: + config_file = path / 'config.json' + if config_file.is_file(): + with open(config_file, encoding = 'utf-8') as f: + prefer_chat_template_alt = json.load(f).get('model_type') == 'cohere2_moe' + chat_template = chat_template_alt if prefer_chat_template_alt else tokenizer_config.get('chat_template', chat_template_alt) if chat_template is None or isinstance(chat_template, (str, list)): self.chat_template = chat_template else: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 18856bedbc..7f25bf82ce 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,6 +109,7 @@ add_library(llama graphs/build_glm4.cpp graphs/build_bitnet.cpp graphs/build_cohere2.cpp + graphs/build_cohere2_moe.cpp graphs/build_t5.cpp graphs/build_jais.cpp graphs/build_chatglm.cpp diff --git a/src/graphs/build_cohere2_moe.cpp b/src/graphs/build_cohere2_moe.cpp new file mode 100644 index 0000000000..05492a5e66 --- /dev/null +++ b/src/graphs/build_cohere2_moe.cpp @@ -0,0 +1,82 @@ +#include "../llama-build-context.h" +#include "../llama-model.h" +#include "../llama-context.h" + +ggml_cgraph * llm_build_context::build_cohere2_moe() { + ggml_cgraph * gf = new_graph_custom(); + + const int64_t n_embd_head = hparams.n_embd_head_v(0); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k(0)); + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + ggml_tensor * inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * KQ_mask = build_inp_KQ_mask(); + ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa(); + + for (int il = 0; il < n_layer; ++il) { + const bool is_sliding = hparams.swa_layers[il]; + const bool force_rope = il < (int) hparams.n_layer_dense_lead; + ggml_tensor * KQ_mask_l = is_sliding ? KQ_mask_swa : KQ_mask; + + ggml_tensor * attn_out = build_std_attention(gf, model.layers[il].attn_norm, inpL, inp_pos, nullptr, nullptr, + KQ_mask_l, nullptr, nullptr, kq_scale, 0.f, + is_sliding ? hparams.n_swa : 0, il, is_sliding || force_rope, false, true, false); + cb(attn_out, "attn_out", il); + + if (il == n_layer - 1 && n_tokens > 1) { + ggml_tensor * inp_out_ids = build_inp_out_ids(); + attn_out = ggml_get_rows(ctx0, attn_out, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + ggml_tensor * cur; + if (model.layers[il].ffn_gate_inp == nullptr) { + attn_out->op_params[3] = 1; + cur = llm_build_ffn(ctx0, lctx, model.layers[il].attn_norm, inpL, + model.layers[il].ffn_up, nullptr, nullptr, + model.layers[il].ffn_gate, nullptr, nullptr, + model.layers[il].ffn_down, nullptr, nullptr, + nullptr, LLM_FFN_SILU, LLM_FFN_PAR, + cb, il, gf, false, false, attn_out); + } else { + cur = llm_build_std_moe_ffn(ctx0, lctx, model.layers[il].attn_norm, inpL, + model.layers[il].ffn_gate_inp, nullptr, + model.layers[il].ffn_up_exps, nullptr, + model.layers[il].ffn_gate_exps, nullptr, + model.layers[il].ffn_down_exps, nullptr, + nullptr, + nullptr, nullptr, + nullptr, nullptr, + nullptr, nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, false, 0.0f, + (llm_expert_gating_func_type) hparams.expert_gating_func, + LLM_FFN_SILU, cb, il, gf, false, model.layers[il].ffn_up_gate_exps, nullptr, nullptr); + cur = ggml_add(ctx0, cur, attn_out); + } + cb(cur, "ffn_out", il); + + cur = lctx.cvec.apply_to(ctx0, cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + ggml_tensor * cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, nullptr, LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + if (hparams.f_logit_scale) { + cur = ggml_scale(ctx0, cur, hparams.f_logit_scale); + cb(cur, "result_norm_scaled", -1); + } + + cur = llm_build_lora_mm(lctx, ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index b93fb66b72..0a402fd1a5 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -64,6 +64,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_GRANITE, "granite" }, { LLM_ARCH_GRANITE_MOE, "granitemoe" }, { LLM_ARCH_COHERE2, "cohere2" }, + { LLM_ARCH_COHERE2_MOE, "cohere2_moe" }, { LLM_ARCH_DOTS1, "dots1" }, { LLM_ARCH_ERNIE4_5, "ernie4_5" }, { LLM_ARCH_ERNIE4_5_MOE, "ernie4_5-moe" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 02a4b996e8..da57b1afb5 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -63,6 +63,7 @@ enum llm_arch { LLM_ARCH_GRANITE, LLM_ARCH_GRANITE_MOE, LLM_ARCH_COHERE2, + LLM_ARCH_COHERE2_MOE, LLM_ARCH_DOTS1, LLM_ARCH_ERNIE4_5, LLM_ARCH_ERNIE4_5_MOE, diff --git a/src/llama-build-context.cpp b/src/llama-build-context.cpp index e7dfa9666a..feba2eaadd 100644 --- a/src/llama-build-context.cpp +++ b/src/llama-build-context.cpp @@ -1595,6 +1595,7 @@ static ggml_tensor * llm_build_kqv( || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_COHERE2 + || model.arch == LLM_ARCH_COHERE2_MOE || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE @@ -1738,7 +1739,7 @@ static ggml_tensor * llm_build_kqv( auto q_i = ggml_view_3d(ctx, q, q->ne[0], q->ne[1], this_ne12, q->nb[1], q->nb[2], q->nb[2]*i12); auto kq_i = ggml_mul_mat(ctx, k_i, q_i); if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || - model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) { + model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_COHERE2_MOE || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) { ggml_mul_mat_set_prec(kq_i, GGML_PREC_F32); } if (model.arch == LLM_ARCH_GROK) { @@ -2448,6 +2449,10 @@ ggml_cgraph * llm_build_context::llama_build_graph( { result = llm.build_cohere2(); } break; + case LLM_ARCH_COHERE2_MOE: + { + result = llm.build_cohere2_moe(); + } break; case LLM_ARCH_T5: { if (lctx.is_encoding) { @@ -2572,6 +2577,7 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_COHERE2 + || model.arch == LLM_ARCH_COHERE2_MOE || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 // || model.arch == LLM_ARCH_GLM4_MOE diff --git a/src/llama-build-context.h b/src/llama-build-context.h index 76f2c71405..7afc5818bf 100644 --- a/src/llama-build-context.h +++ b/src/llama-build-context.h @@ -287,6 +287,7 @@ struct llm_build_context { ggml_cgraph * build_bitnet_158(); ggml_cgraph * build_cohere2(); + ggml_cgraph * build_cohere2_moe(); ggml_cgraph * build_t5_encoder(); diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 8101dad7af..f1a4604696 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -1170,6 +1170,24 @@ void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_COHERE2_MOE: + { + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer); + ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale); + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + if (hparams.expert_gating_func == LLM_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLM_EXPERT_GATING_FUNC_SIGMOID; + } + switch (hparams.n_layer) { + case 49: model.type = e_model::MODEL_30B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; case LLM_ARCH_BAILINGMOE2: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 2522d4cdba..3e8a799208 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -137,6 +137,7 @@ struct create_tensors_helper : public create_tensors_helper_interface { bool create_chatglm_tensors(const LLM_TN & tn); bool create_cohere2_tensors(const LLM_TN & tn); + bool create_cohere2_moe_tensors(const LLM_TN & tn); bool create_dots1_tensors(const LLM_TN & tn); @@ -3213,6 +3214,42 @@ bool create_tensors_helper::create_cohere2_tensors(const LLM_TN & tn) { return use_mmap_buffer; } +bool create_tensors_helper::create_cohere2_moe_tensors(const LLM_TN & tn) { + LOADING_PRELUDE + + model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + model.output = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); + if (model.output == nullptr) { + model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + ggml_context * ctx_split = ctx_for_layer_split(i); + + layer.attn_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0); + + layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}); + layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}); + layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}); + layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}); + + layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd_head_k * n_head}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + + if (i < (int) hparams.n_layer_dense_lead) { + create_std_ffn(i, tn, layer, n_ff, n_embd, ctx_split); + } else { + layer.ffn_gate_inp = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + use_mmap_buffer &= !create_std_ffn_exps(n_embd, tn, i, 0, hparams.n_ff_exp); + } + } + return use_mmap_buffer; +} + bool create_tensors_helper::create_glm4_tensors(const LLM_TN & tn) { LOADING_PRELUDE model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); @@ -4358,6 +4395,8 @@ bool create_tensors_helper::create_tensors() { use_mmap_buffer = create_chatglm_tensors(tn); break; case LLM_ARCH_COHERE2: use_mmap_buffer = create_cohere2_tensors(tn); break; + case LLM_ARCH_COHERE2_MOE: + use_mmap_buffer = create_cohere2_moe_tensors(tn); break; case LLM_ARCH_GLM4: use_mmap_buffer = create_glm4_tensors(tn); break; case LLM_ARCH_DOTS1: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2be2074f57..9617fd54f8 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1340,6 +1340,27 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_COHERE2_MOE, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" }, + { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" }, + { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" }, + { LLM_TENSOR_FFN_GATE_UP_EXPS,"blk.%d.ffn_gate_up_exps" }, + }, + }, { LLM_ARCH_DOTS1, { diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 43ee758f98..3329668f14 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2020,7 +2020,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "refact") { pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT; } else if ( - tokenizer_pre == "command-r") { + tokenizer_pre == "command-r" || + tokenizer_pre == "cohere2_moe") { pre_type = LLAMA_VOCAB_PRE_TYPE_COMMAND_R; clean_spaces = false; } else if ( diff --git a/src/llama.cpp b/src/llama.cpp index fa0f8c1cfa..a84c4828a6 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -3061,6 +3061,7 @@ static bool is_model_split_supported(const llama_model & model) { LLM_ARCH_MISTRAL3, LLM_ARCH_COMMAND_R, LLM_ARCH_COHERE2, + LLM_ARCH_COHERE2_MOE, LLM_ARCH_MIMO2, LLM_ARCH_QWEN3, LLM_ARCH_QWEN3VL, @@ -7395,6 +7396,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_GRANITE: case LLM_ARCH_GRANITE_MOE: case LLM_ARCH_COHERE2: + case LLM_ARCH_COHERE2_MOE: case LLM_ARCH_ERNIE4_5: case LLM_ARCH_ERNIE4_5_MOE: case LLM_ARCH_SMOLLM3: