diff --git a/src/transformers/integrations/ggml.py b/src/transformers/integrations/ggml.py index 9fb285ca343b..fc7c7da89df9 100644 --- a/src/transformers/integrations/ggml.py +++ b/src/transformers/integrations/ggml.py @@ -337,6 +337,29 @@ "vocab_size": "vocab_size", "expert_gating_func": "scoring_func", }, + "glm4moe": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": None, + "rope.freq_base": "rope_theta", + # NOTE: partial rotary (partial_rotary_factor=0.5) keeps its config default; + # attention.key_length carries the full head_dim. + "attention.key_length": "head_dim", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "expert_count": "n_routed_experts", + "expert_used_count": "num_experts_per_tok", + "expert_shared_count": "n_shared_experts", + "expert_feed_forward_length": "moe_intermediate_size", + "expert_group_count": "n_group", + "expert_group_used_count": "topk_group", + "expert_weights_scale": "routed_scaling_factor", + "leading_dense_block_count": "first_k_dense_replace", + "vocab_size": "vocab_size", + }, } GGUF_TOKENIZER_MAPPING = { @@ -824,6 +847,7 @@ def converted(self) -> Tokenizer: "deci": GGUFLlamaConverter, "decilm": GGUFLlamaConverter, "minimax_m2": GGUFQwen2Converter, + "glm4_moe": GGUFGPTConverter, } diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 7f0ebcbb09ba..864a7dde9557 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -453,6 +453,55 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st out.copy_(torch_weights) +class Glm4MoeTensorProcessor(TensorProcessor): + HF_MOE_W13_PATTERN = re.compile(r"(?:model\.)?layers\.(?P\d+)\.mlp\.experts\.gate_up_proj") + GGUF_MOE_WEIGHTS_PATTERN = re.compile(r"(?P.*\.ffn_(?Pgate|down|up)_exps)\.weight$") + HF_BIAS_PATTERN = re.compile(r"(?:model\.)?layers\.(?P\d+)\.mlp\.gate\.e_score_correction_bias") + + def __init__(self, config=None): + super().__init__(config=config) + + def perform_fallback_tensor_mapping( + self, gguf_to_hf_name_map: dict[str, str], suffix: str, qual_name: str, hf_name: str + ): + # Routed experts store gate (w1) and up (w3) as one merged gate_up_proj on the HF side. + if m := re.fullmatch(self.HF_MOE_W13_PATTERN, hf_name): + full_hf_name = qual_name + hf_name + gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_gate_exps{suffix}"] = full_hf_name + gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_up_exps{suffix}"] = full_hf_name + # Routing bias is stored under mlp.gate on the HF side. + elif m := re.fullmatch(self.HF_BIAS_PATTERN, hf_name): + gguf_to_hf_name_map[f"blk.{m['bid']}.exp_probs_b.bias"] = qual_name + hf_name + + def process(self, weights, name: str, **kwargs): + if m := re.fullmatch(self.GGUF_MOE_WEIGHTS_PATTERN, name): + tensor_key_mapping = kwargs.get("tensor_key_mapping") + parsed_parameters = kwargs.get("parsed_parameters") + if tensor_key_mapping: + self._set_moe_expert_tensor(weights, parsed_parameters, tensor_key_mapping[m["name"]], m["w"]) + return GGUFTensor(weights, None, {}) + return GGUFTensor(weights, name, {}) + + def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[str, dict], hf_name: str, w: str): + torch_weights = torch.from_numpy(np.copy(weights)) + if w == "down": + parsed_parameters["tensors"][hf_name] = torch_weights + else: + # Merge gate and up into gate_up_proj [num_experts, 2*moe_intermediate, hidden] + shape = list(weights.shape) + shard_dim = 1 + shard_size = shape[shard_dim] + shape[shard_dim] = shard_size * 2 + if hf_name not in parsed_parameters["tensors"]: + parsed_parameters["tensors"][hf_name] = torch.zeros(shape, dtype=torch_weights.dtype) + out: torch.Tensor = parsed_parameters["tensors"][hf_name] + if w == "gate": + out = out.narrow(shard_dim, 0, shard_size) + else: # w == "up" + out = out.narrow(shard_dim, shard_size, shard_size) + out.copy_(torch_weights) + + TENSOR_PROCESSORS = { "llama": LlamaTensorProcessor, "qwen2moe": Qwen2MoeTensorProcessor, @@ -468,6 +517,7 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st "gemma3": Gemma2TensorProcessor, "lfm2": Lfm2TensorProcessor, "minimax-m2": MiniMaxM2TensorProcessor, + "glm4moe": Glm4MoeTensorProcessor, } @@ -522,6 +572,8 @@ def get_gguf_hf_weights_map( model_type = "minimax-m2" elif model_type == "gpt_oss": model_type = "gpt-oss" + elif model_type == "glm4_moe": + model_type = "glm4moe" arch = None for key, value in MODEL_ARCH_NAMES.items(): if value == model_type: @@ -693,6 +745,18 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo if gguf_key in reader_keys: logger.info(f"Some keys were not parsed and added into account {gguf_key} | {value}") + # GGUF uses `glm4moe` as architecture name while Transformers uses `glm4_moe` + if parsed_parameters["config"]["model_type"] == "glm4moe": + parsed_parameters["config"]["model_type"] = "glm4_moe" + # block_count includes Multi-Token-Prediction (nextn) layers, which the HF model does not represent. + n_nextn = read_field(reader, "glm4moe.nextn_predict_layers") + if n_nextn and parsed_parameters["config"].get("num_hidden_layers") is not None: + parsed_parameters["config"]["num_hidden_layers"] -= int(n_nextn[0]) + # QKV bias and QK-norm are not stored as metadata; infer them from tensor presence. + tensor_names = {tensor.name for tensor in reader.tensors} + parsed_parameters["config"]["attention_bias"] = any(name.endswith("attn_q.bias") for name in tensor_names) + parsed_parameters["config"]["use_qk_norm"] = any("attn_q_norm" in name for name in tensor_names) + # Gemma3 GGUF checkpoint only contains weights of text backbone if parsed_parameters["config"]["model_type"] == "gemma3": parsed_parameters["config"]["model_type"] = "gemma3_text" diff --git a/tests/quantization/ggml/test_ggml.py b/tests/quantization/ggml/test_ggml.py index aa5cdbc7adc6..b311e4176623 100644 --- a/tests/quantization/ggml/test_ggml.py +++ b/tests/quantization/ggml/test_ggml.py @@ -311,6 +311,8 @@ class GgufModelTests(unittest.TestCase): qwen3moe_model_id = "Qwen/Qwen3-30B-A3B-GGUF" umt5_encoder_model_id = "city96/umt5-xxl-encoder-gguf" lfm2_model_id = "LiquidAI/LFM2-1.2B-GGUF" + glm4moe_model_id = "mironableee/tiny-glm4moe-gguf" + glm4moe_original_model_id = "trl-internal-testing/tiny-Glm4MoeForCausalLM" q4_0_phi3_model_id = "Phi-3-mini-4k-instruct-q4.gguf" q4_0_mistral_model_id = "mistral-7b-instruct-v0.2.Q4_0.gguf" @@ -328,6 +330,7 @@ class GgufModelTests(unittest.TestCase): fp16_t5_model_id = "flan-t5-small.F16.gguf" q8_0_t5_model_id = "flan-t5-small.Q8_0.gguf" fp16_qwen2moe_model_id = "Qwen1.5-MoE-A2.7B.gguf" + f16_glm4moe_gguf_file = "tiny-glm4moe-f16.gguf" fp16_gpt2_model_id = "gpt2.f16.gguf" q8_gpt2_model_id = "gpt2.Q8_0.gguf" q6_k_gpt2_xl_model_id = "gpt2-xl.Q6_K.gguf" @@ -1145,3 +1148,22 @@ def test_lfm2_q4_k_m(self): EXPECTED_TEXT = "Hello Atari 2600! es un videoj" self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT) + + @unittest.skipUnless(is_gguf_available("0.19.0"), "GLM4_MOE requires gguf version >= 0.19.0") + def test_glm4moe_f16(self): + # The fixture is a tiny-random model, so we check load fidelity against the original weights + # (covers the dense layer, merged routed experts, shared experts and routing bias paths) + # rather than asserting decoded text. + model = AutoModelForCausalLM.from_pretrained( + self.glm4moe_model_id, + gguf_file=self.f16_glm4moe_gguf_file, + dtype=torch.float16, + ) + original_model = AutoModelForCausalLM.from_pretrained(self.glm4moe_original_model_id, dtype=torch.float16) + + text = torch.tensor([[1, 2, 3, 4, 5]]) + with torch.no_grad(): + logits = model(text).logits + original_logits = original_model(text).logits + + torch.testing.assert_close(logits, original_logits, rtol=1e-3, atol=1e-3)