diff --git a/src/transformers/integrations/ggml.py b/src/transformers/integrations/ggml.py index 29ec365e7ce2..eeffc9d3a97d 100644 --- a/src/transformers/integrations/ggml.py +++ b/src/transformers/integrations/ggml.py @@ -89,6 +89,27 @@ "expert_count": "num_experts", "expert_used_count": "num_experts_per_tok", }, + "qwen35": { + "context_length": "max_position_embeddings", + "block_count": "num_hidden_layers", + "feed_forward_length": "intermediate_size", + "embedding_length": "hidden_size", + "rope.dimension_count": "_rope_dimension_count", + "rope.dimension_sections": "_rope_dimension_sections", + "rope.freq_base": "rope_theta", + "attention.head_count": "num_attention_heads", + "attention.head_count_kv": "num_key_value_heads", + "attention.key_length": "head_dim", + "attention.value_length": None, + "attention.layer_norm_rms_epsilon": "rms_norm_eps", + "vocab_size": "vocab_size", + "full_attention_interval": "full_attention_interval", + "ssm.state_size": "linear_key_head_dim", + "ssm.conv_kernel": "linear_conv_kernel_dim", + "ssm.inner_size": "ssm_inner_size", + "ssm.group_count": "linear_num_key_heads", + "ssm.time_step_rank": "linear_num_value_heads", + }, "lfm2": { "context_length": "max_position_embeddings", "block_count": "num_hidden_layers", @@ -776,6 +797,8 @@ def converted(self) -> Tokenizer: "qwen2_moe": GGUFQwen2Converter, "qwen3": GGUFQwen2Converter, "qwen3_moe": GGUFQwen2Converter, + "qwen35": GGUFQwen2Converter, + "qwen3_5_text": GGUFQwen2Converter, "phi3": GGUFPhi3Converter, "bloom": GGUFGPTConverter, "falcon": GGUFGPTConverter, diff --git a/src/transformers/modeling_gguf_pytorch_utils.py b/src/transformers/modeling_gguf_pytorch_utils.py index 66306b6f71f6..e1d0af5e479d 100644 --- a/src/transformers/modeling_gguf_pytorch_utils.py +++ b/src/transformers/modeling_gguf_pytorch_utils.py @@ -265,6 +265,81 @@ def process(self, weights, name, **kwargs): return GGUFTensor(weights, name, {}) +class Qwen35TensorProcessor(TensorProcessor): + """Qwen3.5 tensor processor for SSM transforms and linear-attention head reordering.""" + + def __init__(self, config=None): + super().__init__(config=config) + self.num_k_heads = self.config.get("linear_num_key_heads") + self.num_v_heads = self.config.get("linear_num_value_heads") + self.head_k_dim = self.config.get("linear_key_head_dim") + self.head_v_dim = self.config.get("linear_value_head_dim") + if self.head_v_dim is None and None not in (self.config.get("ssm_inner_size"), self.num_v_heads): + self.head_v_dim = self.config["ssm_inner_size"] // self.num_v_heads + + def _reverse_reorder_v_heads(self, weights: np.ndarray, dim: int, head_dim: int) -> np.ndarray: + if None in (self.num_k_heads, self.num_v_heads, head_dim) or self.num_k_heads == self.num_v_heads: + return weights + + num_v_per_k = self.num_v_heads // self.num_k_heads + if dim < 0: + dim += weights.ndim + + shape = list(weights.shape) + new_shape = shape[:dim] + [num_v_per_k, self.num_k_heads, head_dim] + shape[dim + 1 :] + weights = weights.reshape(new_shape) + weights = weights.swapaxes(dim, dim + 1) + return weights.reshape(shape) + + def process(self, weights, name, **kwargs): + # SSM-specific transforms + if "norm.weight" in name and "ssm_norm.weight" not in name: + weights = weights - 1 + if "attn_qkv.weight" in name and None not in ( + self.num_k_heads, + self.num_v_heads, + self.head_k_dim, + self.head_v_dim, + ): + q_dim = self.head_k_dim * self.num_k_heads + k_dim = self.head_k_dim * self.num_k_heads + q = weights[:q_dim] + k = weights[q_dim : q_dim + k_dim] + v = weights[q_dim + k_dim :] + v = self._reverse_reorder_v_heads(v, 0, self.head_v_dim) + weights = np.concatenate([q, k, v], axis=0) + elif "attn_gate.weight" in name: + weights = self._reverse_reorder_v_heads(weights, 0, self.head_v_dim) + elif "ssm_alpha.weight" in name or "ssm_beta.weight" in name: + weights = self._reverse_reorder_v_heads(weights, 0, 1) + elif "ssm_out.weight" in name: + weights = self._reverse_reorder_v_heads(weights, 1, self.head_v_dim) + if "ssm_conv1d.weight" in name: + if None not in (self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim): + qk_channels = self.head_k_dim * self.num_k_heads * 2 + qk_part = weights[:qk_channels] + v_part = self._reverse_reorder_v_heads(weights[qk_channels:], 0, self.head_v_dim) + weights = np.concatenate([qk_part, v_part], axis=0) + # GGUF shape is (inner_size, conv_kernel), HF expects (inner_size, 1, conv_kernel) + weights = np.expand_dims(weights, axis=1) + if name.endswith("ssm_a.weight") or name.endswith("ssm_a"): + # llama.cpp stores -exp(A_log), need to convert back to A_log + weights = np.log(-weights) + weights = self._reverse_reorder_v_heads(weights, 0, 1) + if "ssm_dt.bias" in name: + # GGUF maps ssm_dt.bias -> linear_attn.dt_proj.bias, but HF model uses linear_attn.dt_bias + tensor_key_mapping = kwargs.get("tensor_key_mapping") + parsed_parameters = kwargs.get("parsed_parameters") + if tensor_key_mapping is not None and parsed_parameters is not None: + parts = name.split(".") + bid = parts[1] + hf_name = f"model.layers.{bid}.linear_attn.dt_bias" + weights = self._reverse_reorder_v_heads(weights, 0, 1) + parsed_parameters["tensors"][hf_name] = torch.from_numpy(np.copy(weights)) + return GGUFTensor(weights, None, {}) + return GGUFTensor(weights, name, {}) + + class NemotronTensorProcessor(TensorProcessor): def __init__(self, config=None): super().__init__(config=config) @@ -361,6 +436,7 @@ def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[st "t5encoder": T5TensorProcessor, "gpt2": GPT2TensorProcessor, "mamba": MambaTensorProcessor, + "qwen35": Qwen35TensorProcessor, "nemotron": NemotronTensorProcessor, "gemma2": Gemma2TensorProcessor, "gemma3": Gemma2TensorProcessor, @@ -416,6 +492,8 @@ def get_gguf_hf_weights_map( model_type = "t5" elif model_type == "minimax_m2": model_type = "minimax-m2" + elif model_type in ("qwen3_5", "qwen3_5_text"): + model_type = "qwen35" arch = None for key, value in MODEL_ARCH_NAMES.items(): if value == model_type: @@ -520,6 +598,8 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo updated_architecture = "qwen3_moe" elif "minimax-m2" in architecture: updated_architecture = "minimax_m2" + elif "qwen35" in architecture: + updated_architecture = "qwen35" # For stablelm architecture, we need to set qkv_bias and use_parallel_residual from tensors # If `qkv_bias=True`, qkv_proj with bias will be present in the tensors @@ -583,6 +663,10 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo if parsed_parameters["config"]["model_type"] == "gemma3": parsed_parameters["config"]["model_type"] = "gemma3_text" + # Qwen3.5 GGUF checkpoint only contains weights of text backbone + if parsed_parameters["config"].get("model_type") == "qwen35": + parsed_parameters["config"]["model_type"] = "qwen3_5_text" + # MiniMax-M2: convert expert_gating_func integer to scoring_func string if parsed_parameters["config"].get("model_type") == "minimax_m2": _gating_func_map = {0: "none", 1: "softmax", 2: "sigmoid"}