Add GGUF loading support for GLM-4.5 MoE (glm4moe)

src/transformers/integrations/ggml.py

@@ -337,6 +337,29 @@
         "vocab_size": "vocab_size",
         "expert_gating_func": "scoring_func",
     },
+    "glm4moe": {
+        "context_length": "max_position_embeddings",
+        "block_count": "num_hidden_layers",
+        "feed_forward_length": "intermediate_size",
+        "embedding_length": "hidden_size",
+        "rope.dimension_count": None,
+        "rope.freq_base": "rope_theta",
+        # NOTE: partial rotary (partial_rotary_factor=0.5) keeps its config default;
+        # attention.key_length carries the full head_dim.
+        "attention.key_length": "head_dim",
+        "attention.head_count": "num_attention_heads",
+        "attention.head_count_kv": "num_key_value_heads",
+        "attention.layer_norm_rms_epsilon": "rms_norm_eps",
+        "expert_count": "n_routed_experts",
+        "expert_used_count": "num_experts_per_tok",
+        "expert_shared_count": "n_shared_experts",
+        "expert_feed_forward_length": "moe_intermediate_size",
+        "expert_group_count": "n_group",
+        "expert_group_used_count": "topk_group",
+        "expert_weights_scale": "routed_scaling_factor",
+        "leading_dense_block_count": "first_k_dense_replace",
+        "vocab_size": "vocab_size",
+    },
 }
 
 GGUF_TOKENIZER_MAPPING = {
@@ -824,6 +847,7 @@ def converted(self) -> Tokenizer:
     "deci": GGUFLlamaConverter,
     "decilm": GGUFLlamaConverter,
     "minimax_m2": GGUFQwen2Converter,
+    "glm4_moe": GGUFGPTConverter,
 }
 
 

src/transformers/modeling_gguf_pytorch_utils.py

@@ -453,6 +453,55 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st
             out.copy_(torch_weights)
 
 
+class Glm4MoeTensorProcessor(TensorProcessor):
+    HF_MOE_W13_PATTERN = re.compile(r"(?:model\.)?layers\.(?P<bid>\d+)\.mlp\.experts\.gate_up_proj")
+    GGUF_MOE_WEIGHTS_PATTERN = re.compile(r"(?P<name>.*\.ffn_(?P<w>gate|down|up)_exps)\.weight$")
+    HF_BIAS_PATTERN = re.compile(r"(?:model\.)?layers\.(?P<bid>\d+)\.mlp\.gate\.e_score_correction_bias")
+
+    def __init__(self, config=None):
+        super().__init__(config=config)
+
+    def perform_fallback_tensor_mapping(
+        self, gguf_to_hf_name_map: dict[str, str], suffix: str, qual_name: str, hf_name: str
+    ):
+        # Routed experts store gate (w1) and up (w3) as one merged gate_up_proj on the HF side.
+        if m := re.fullmatch(self.HF_MOE_W13_PATTERN, hf_name):
+            full_hf_name = qual_name + hf_name
+            gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_gate_exps{suffix}"] = full_hf_name
+            gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_up_exps{suffix}"] = full_hf_name
+        # Routing bias is stored under mlp.gate on the HF side.
+        elif m := re.fullmatch(self.HF_BIAS_PATTERN, hf_name):
+            gguf_to_hf_name_map[f"blk.{m['bid']}.exp_probs_b.bias"] = qual_name + hf_name
+
+    def process(self, weights, name: str, **kwargs):
+        if m := re.fullmatch(self.GGUF_MOE_WEIGHTS_PATTERN, name):
+            tensor_key_mapping = kwargs.get("tensor_key_mapping")
+            parsed_parameters = kwargs.get("parsed_parameters")
+            if tensor_key_mapping:
+                self._set_moe_expert_tensor(weights, parsed_parameters, tensor_key_mapping[m["name"]], m["w"])
+            return GGUFTensor(weights, None, {})
+        return GGUFTensor(weights, name, {})
+
+    def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[str, dict], hf_name: str, w: str):
+        torch_weights = torch.from_numpy(np.copy(weights))
+        if w == "down":
+            parsed_parameters["tensors"][hf_name] = torch_weights
+        else:
+            # Merge gate and up into gate_up_proj [num_experts, 2*moe_intermediate, hidden]
+            shape = list(weights.shape)
+            shard_dim = 1
+            shard_size = shape[shard_dim]
+            shape[shard_dim] = shard_size * 2
+            if hf_name not in parsed_parameters["tensors"]:
+                parsed_parameters["tensors"][hf_name] = torch.zeros(shape, dtype=torch_weights.dtype)
+            out: torch.Tensor = parsed_parameters["tensors"][hf_name]
+            if w == "gate":
+                out = out.narrow(shard_dim, 0, shard_size)
+            else:  # w == "up"
+                out = out.narrow(shard_dim, shard_size, shard_size)
+            out.copy_(torch_weights)
+
+
 TENSOR_PROCESSORS = {
     "llama": LlamaTensorProcessor,
     "qwen2moe": Qwen2MoeTensorProcessor,
@@ -468,6 +517,7 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st
     "gemma3": Gemma2TensorProcessor,
     "lfm2": Lfm2TensorProcessor,
     "minimax-m2": MiniMaxM2TensorProcessor,
+    "glm4moe": Glm4MoeTensorProcessor,
 }
 
 
@@ -522,6 +572,8 @@ def get_gguf_hf_weights_map(
         model_type = "minimax-m2"
     elif model_type == "gpt_oss":
         model_type = "gpt-oss"
+    elif model_type == "glm4_moe":
+        model_type = "glm4moe"
     arch = None
     for key, value in MODEL_ARCH_NAMES.items():
         if value == model_type:
@@ -693,6 +745,18 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
         if gguf_key in reader_keys:
             logger.info(f"Some keys were not parsed and added into account {gguf_key} | {value}")
 
+    # GGUF uses `glm4moe` as architecture name while Transformers uses `glm4_moe`
+    if parsed_parameters["config"]["model_type"] == "glm4moe":
+        parsed_parameters["config"]["model_type"] = "glm4_moe"
+        # block_count includes Multi-Token-Prediction (nextn) layers, which the HF model does not represent.
+        n_nextn = read_field(reader, "glm4moe.nextn_predict_layers")
+        if n_nextn and parsed_parameters["config"].get("num_hidden_layers") is not None:
+            parsed_parameters["config"]["num_hidden_layers"] -= int(n_nextn[0])
+        # QKV bias and QK-norm are not stored as metadata; infer them from tensor presence.
+        tensor_names = {tensor.name for tensor in reader.tensors}
+        parsed_parameters["config"]["attention_bias"] = any(name.endswith("attn_q.bias") for name in tensor_names)
+        parsed_parameters["config"]["use_qk_norm"] = any("attn_q_norm" in name for name in tensor_names)
+
     # Gemma3 GGUF checkpoint only contains weights of text backbone
     if parsed_parameters["config"]["model_type"] == "gemma3":
         parsed_parameters["config"]["model_type"] = "gemma3_text"

tests/quantization/ggml/test_ggml.py

@@ -311,6 +311,8 @@ class GgufModelTests(unittest.TestCase):
     qwen3moe_model_id = "Qwen/Qwen3-30B-A3B-GGUF"
     umt5_encoder_model_id = "city96/umt5-xxl-encoder-gguf"
     lfm2_model_id = "LiquidAI/LFM2-1.2B-GGUF"
+    glm4moe_model_id = "mironableee/tiny-glm4moe-gguf"
+    glm4moe_original_model_id = "trl-internal-testing/tiny-Glm4MoeForCausalLM"
 
     q4_0_phi3_model_id = "Phi-3-mini-4k-instruct-q4.gguf"
     q4_0_mistral_model_id = "mistral-7b-instruct-v0.2.Q4_0.gguf"
@@ -328,6 +330,7 @@ class GgufModelTests(unittest.TestCase):
     fp16_t5_model_id = "flan-t5-small.F16.gguf"
     q8_0_t5_model_id = "flan-t5-small.Q8_0.gguf"
     fp16_qwen2moe_model_id = "Qwen1.5-MoE-A2.7B.gguf"
+    f16_glm4moe_gguf_file = "tiny-glm4moe-f16.gguf"
     fp16_gpt2_model_id = "gpt2.f16.gguf"
     q8_gpt2_model_id = "gpt2.Q8_0.gguf"
     q6_k_gpt2_xl_model_id = "gpt2-xl.Q6_K.gguf"
@@ -1145,3 +1148,22 @@ def test_lfm2_q4_k_m(self):
 
         EXPECTED_TEXT = "Hello Atari 2600! es un videoj"
         self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
+
+    @unittest.skipUnless(is_gguf_available("0.19.0"), "GLM4_MOE requires gguf version >= 0.19.0")
+    def test_glm4moe_f16(self):
+        # The fixture is a tiny-random model, so we check load fidelity against the original weights
+        # (covers the dense layer, merged routed experts, shared experts and routing bias paths)
+        # rather than asserting decoded text.
+        model = AutoModelForCausalLM.from_pretrained(
+            self.glm4moe_model_id,
+            gguf_file=self.f16_glm4moe_gguf_file,
+            dtype=torch.float16,
+        )
+        original_model = AutoModelForCausalLM.from_pretrained(self.glm4moe_original_model_id, dtype=torch.float16)
+
+        text = torch.tensor([[1, 2, 3, 4, 5]])
+        with torch.no_grad():
+            logits = model(text).logits
+            original_logits = original_model(text).logits
+
+        torch.testing.assert_close(logits, original_logits, rtol=1e-3, atol=1e-3)