From c84d27927e8510d57726b0de74d2e74d740923e5 Mon Sep 17 00:00:00 2001 From: Curnane Date: Wed, 15 Jul 2026 11:27:30 +0800 Subject: [PATCH 1/7] Add num_mtp_layers config attribute --- src/transformers/models/qwen3_5/configuration_qwen3_5.py | 7 +++++++ src/transformers/models/qwen3_5/modular_qwen3_5.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/transformers/models/qwen3_5/configuration_qwen3_5.py b/src/transformers/models/qwen3_5/configuration_qwen3_5.py index c33663287cea..5b22f65e24d8 100644 --- a/src/transformers/models/qwen3_5/configuration_qwen3_5.py +++ b/src/transformers/models/qwen3_5/configuration_qwen3_5.py @@ -38,6 +38,8 @@ class Qwen3_5TextConfig(PreTrainedConfig): Number of key heads used in linear attention layers. linear_num_value_heads (`int`, *optional*, defaults to 32): Number of value heads used in linear attention layers. + num_mtp_layers (`int`, *optional*, defaults to 0): + Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -106,6 +108,7 @@ class Qwen3_5TextConfig(PreTrainedConfig): eos_token_id: int | list[int] | None = None base_config_key = "text_config" ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} + num_mtp_layers: int = 0 def __post_init__(self, **kwargs): kwargs.setdefault("partial_rotary_factor", 0.25) # assign default for BC @@ -152,6 +155,9 @@ class Qwen3_5VisionConfig(PreTrainedConfig): @strict class Qwen3_5Config(PreTrainedConfig): r""" + num_mtp_layers (`int`, *optional*, defaults to 0): + Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + Example: ```python @@ -179,6 +185,7 @@ class Qwen3_5Config(PreTrainedConfig): vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 tie_word_embeddings: bool = False + num_mtp_layers: int = 0 def __post_init__(self, **kwargs): if isinstance(self.vision_config, dict): diff --git a/src/transformers/models/qwen3_5/modular_qwen3_5.py b/src/transformers/models/qwen3_5/modular_qwen3_5.py index 8eeda8632712..1df8ae42663c 100644 --- a/src/transformers/models/qwen3_5/modular_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modular_qwen3_5.py @@ -75,6 +75,8 @@ class Qwen3_5TextConfig(Qwen3NextConfig): Number of key heads used in linear attention layers. linear_num_value_heads (`int`, *optional*, defaults to 32): Number of value heads used in linear attention layers. + num_mtp_layers (`int`, *optional*, defaults to 0): + Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -117,6 +119,7 @@ class Qwen3_5TextConfig(Qwen3NextConfig): intermediate_size: int = 12288 num_hidden_layers: int = 32 num_key_value_heads: int = 4 + num_mtp_layers: int = 0 decoder_sparse_step = AttributeError() norm_topk_prob = AttributeError() @@ -150,6 +153,9 @@ class Qwen3_5VisionConfig(Qwen3VLVisionConfig): @strict class Qwen3_5Config(Qwen3VLConfig): r""" + num_mtp_layers (`int`, *optional*, defaults to 0): + Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + Example: ```python @@ -169,6 +175,7 @@ class Qwen3_5Config(Qwen3VLConfig): video_token_id: int = 248057 vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 + num_mtp_layers: int = 0 class Qwen3_5VisionRotaryEmbedding(Qwen3VLVisionRotaryEmbedding): From 1fc43e251b275e08104be5d1ea6f567deabe0f8c Mon Sep 17 00:00:00 2001 From: Curnane Date: Wed, 15 Jul 2026 11:27:32 +0800 Subject: [PATCH 2/7] Adapt MtpModel and MTPCandidateGenerator for Qwen3.5 --- .../generation/candidate_generator.py | 4 +-- src/transformers/modeling_layers.py | 29 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/transformers/generation/candidate_generator.py b/src/transformers/generation/candidate_generator.py index 5776e2ccec98..5e49536cc927 100644 --- a/src/transformers/generation/candidate_generator.py +++ b/src/transformers/generation/candidate_generator.py @@ -1531,7 +1531,7 @@ def get_candidates( # We need the last invalidated tokens, as well as the new one in a single passcfor efficiency mtp_input_ids = input_ids[:, -num_last_main_model_tokens - self.num_mtp_layers :] - mtp_position_ids = model_kwargs["position_ids"][:, -num_last_main_model_tokens - self.num_mtp_layers :] + mtp_position_ids = model_kwargs["position_ids"][..., -num_last_main_model_tokens - self.num_mtp_layers :] mtp_attention_mask = model_kwargs["attention_mask"][:, -num_last_main_model_tokens - self.num_mtp_layers :] last_hidden_states = self.full_seq_last_hidden_states[ :, -num_last_main_model_tokens - self.num_mtp_layers :, : @@ -1539,7 +1539,7 @@ def get_candidates( # If we have a single layer, or validated enough, we can simply pass the new tokens else: mtp_input_ids = input_ids[:, -num_last_main_model_tokens:] - mtp_position_ids = model_kwargs["position_ids"][:, -num_last_main_model_tokens:] + mtp_position_ids = model_kwargs["position_ids"][..., -num_last_main_model_tokens:] mtp_attention_mask = model_kwargs["attention_mask"][:, -num_last_main_model_tokens:] candidate_ids, candidate_logits, _ = self.mtp_model( diff --git a/src/transformers/modeling_layers.py b/src/transformers/modeling_layers.py index 07757ba7471a..90186991fc2c 100644 --- a/src/transformers/modeling_layers.py +++ b/src/transformers/modeling_layers.py @@ -498,18 +498,29 @@ def forward( for i, mtp_layer in enumerate(self.layers): # We need to recompute those every layer since they change inputs_embeds = self.embed_tokens(input_ids).to(last_hidden_states.device) + + # Support both 2D position_ids and MRoPE-style nD ids (e.g. Qwen3.5 uses [4, B, L]) + if position_ids.ndim == 2: + text_position_ids = position_ids + rope_position_ids = position_ids + else: + text_position_ids = position_ids[0] # first component is text positions + rope_position_ids = position_ids[1:] if position_ids.shape[0] > 1 else position_ids + position_embeddings = ( - self.rotary_emb(inputs_embeds, position_ids=position_ids) if self.rotary_emb is not None else None + self.rotary_emb(inputs_embeds, position_ids=rope_position_ids) + if self.rotary_emb is not None + else None ) # In full generality, we may need to recompute masks for every layer due to the position offset of each layer - masks = self.create_masks_for_mtp_layer(i, inputs_embeds, mtp_cache, position_ids) + masks = self.create_masks_for_mtp_layer(i, inputs_embeds, mtp_cache, text_position_ids) last_hidden_states = mtp_layer( inputs_embeds, last_hidden_states, position_embeddings=position_embeddings, - position_ids=position_ids, + position_ids=text_position_ids, past_key_values=mtp_cache, **masks, **kwargs, @@ -545,7 +556,8 @@ def forward( # Roll by 1 and append for next layer input_ids = torch.cat([input_ids[:, 1:], next_mtp_token], dim=-1) attention_mask = torch.cat([attention_mask[:, 1:], attention_mask.new_ones(batch_size, 1)], dim=-1) # type: ignore - position_ids = torch.cat([position_ids[:, 1:], position_ids[:, -1:] + 1], dim=-1) + next_position_ids = position_ids[..., -1:] + 1 + position_ids = torch.cat([position_ids[..., 1:], next_position_ids], dim=-1) # Need to cat ful_ids as well for the processors if full_input_ids is not None: @@ -560,12 +572,15 @@ def from_pretrained(cls, main_model: PreTrainedModel, device_map=None, **kwargs) pretrained_model_name_or_path = main_model.config.name_or_path num_hidden_layers = main_model.config.get_text_config().num_hidden_layers # Heuristic: the main model should have the mtp layer patterns under `_keys_to_ignore_on_load_unexpected` to avoid - # loading them by default, so use it to later load the correct keys from the checkpoints + # loading them by default, so use it to later load the correct keys from the checkpoints. + # Filter out non-MTP patterns (e.g. vision tower weights for multimodal models like Qwen3.5). mtp_patterns = main_model._keys_to_ignore_on_load_unexpected.copy() # type: ignore - # Due to different released checkpoints, only keep the ones with layer number >= num_hidden_layers - otherwise - # mtp layers in a smaller checkpoints could be wrongly added as a 2nd mtp layer of a bigger checkpoint final_mtp_patterns = [] for pattern in mtp_patterns: + has_mtp_keyword = "mtp" in pattern.lower() + has_layer_pattern = re.search(r"layers?\.\d+", pattern, re.IGNORECASE) is not None + if not has_mtp_keyword and not has_layer_pattern: + continue match_object = re.search(r"\.(\d+)", pattern) if match_object is not None and int(match_object.group(1)) < num_hidden_layers: continue From 96def09d06c088d1f82076063d1ddee261e23da9 Mon Sep 17 00:00:00 2001 From: Curnane Date: Thu, 16 Jul 2026 10:36:19 +0800 Subject: [PATCH 3/7] fix(qwen3.5): add mtp_num_hidden_layers and mtp_layer_types for MTP config --- .../models/qwen3_5/configuration_qwen3_5.py | 31 ++++++++++++++--- .../models/qwen3_5/modular_qwen3_5.py | 34 ++++++++++++++++--- tests/models/qwen3_5/test_modeling_qwen3_5.py | 20 +++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/qwen3_5/configuration_qwen3_5.py b/src/transformers/models/qwen3_5/configuration_qwen3_5.py index 5b22f65e24d8..1f40168b71a3 100644 --- a/src/transformers/models/qwen3_5/configuration_qwen3_5.py +++ b/src/transformers/models/qwen3_5/configuration_qwen3_5.py @@ -38,8 +38,11 @@ class Qwen3_5TextConfig(PreTrainedConfig): Number of key heads used in linear attention layers. linear_num_value_heads (`int`, *optional*, defaults to 32): Number of value heads used in linear attention layers. - num_mtp_layers (`int`, *optional*, defaults to 0): + mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + This field is exposed as `num_mtp_layers` via `attribute_map`. + mtp_layer_types (`list[str]`, *optional*): + Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -107,8 +110,14 @@ class Qwen3_5TextConfig(PreTrainedConfig): bos_token_id: int | None = None eos_token_id: int | list[int] | None = None base_config_key = "text_config" + + attribute_map = { + **Qwen3NextConfig.attribute_map, + "num_mtp_layers": "mtp_num_hidden_layers", + } ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} - num_mtp_layers: int = 0 + mtp_num_hidden_layers: int = 0 + mtp_layer_types: list[str] | None = None def __post_init__(self, **kwargs): kwargs.setdefault("partial_rotary_factor", 0.25) # assign default for BC @@ -122,6 +131,8 @@ def __post_init__(self, **kwargs): self.layer_types = remap_legacy_layer_types(self.layer_types) super().__post_init__(**kwargs) + if self.mtp_layer_types is None: + self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers @auto_docstring(checkpoint="Qwen/Qwen3.5-27B") @@ -155,8 +166,11 @@ class Qwen3_5VisionConfig(PreTrainedConfig): @strict class Qwen3_5Config(PreTrainedConfig): r""" - num_mtp_layers (`int`, *optional*, defaults to 0): + mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + This field is exposed as `num_mtp_layers` via `attribute_map`. + mtp_layer_types (`list[str]`, *optional*): + Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. Example: @@ -185,7 +199,14 @@ class Qwen3_5Config(PreTrainedConfig): vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 tie_word_embeddings: bool = False - num_mtp_layers: int = 0 + + attribute_map = { + **Qwen3VLConfig.attribute_map, + "num_mtp_layers": "mtp_num_hidden_layers", + } + + mtp_num_hidden_layers: int = 0 + mtp_layer_types: list[str] | None = None def __post_init__(self, **kwargs): if isinstance(self.vision_config, dict): @@ -202,6 +223,8 @@ def __post_init__(self, **kwargs): self.text_config = self.sub_configs["text_config"]() super().__post_init__(**kwargs) + if self.mtp_layer_types is None: + self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers __all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"] diff --git a/src/transformers/models/qwen3_5/modular_qwen3_5.py b/src/transformers/models/qwen3_5/modular_qwen3_5.py index 1df8ae42663c..f25794ce6555 100644 --- a/src/transformers/models/qwen3_5/modular_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modular_qwen3_5.py @@ -75,8 +75,11 @@ class Qwen3_5TextConfig(Qwen3NextConfig): Number of key heads used in linear attention layers. linear_num_value_heads (`int`, *optional*, defaults to 32): Number of value heads used in linear attention layers. - num_mtp_layers (`int`, *optional*, defaults to 0): + mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + This field is exposed as `num_mtp_layers` via `attribute_map`. + mtp_layer_types (`list[str]`, *optional*): + Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -95,6 +98,11 @@ class Qwen3_5TextConfig(Qwen3NextConfig): model_type = "qwen3_5_text" base_config_key = "text_config" + attribute_map = { + **Qwen3NextConfig.attribute_map, + "num_mtp_layers": "mtp_num_hidden_layers", + } + base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise", "layers.*.self_attn.k_proj": "colwise", @@ -119,7 +127,8 @@ class Qwen3_5TextConfig(Qwen3NextConfig): intermediate_size: int = 12288 num_hidden_layers: int = 32 num_key_value_heads: int = 4 - num_mtp_layers: int = 0 + mtp_num_hidden_layers: int = 0 + mtp_layer_types: list[str] | None = None decoder_sparse_step = AttributeError() norm_topk_prob = AttributeError() @@ -133,6 +142,8 @@ class Qwen3_5TextConfig(Qwen3NextConfig): def __post_init__(self, **kwargs): super().__post_init__(**kwargs) + if self.mtp_layer_types is None: + self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers del self.mlp_only_layers @@ -153,8 +164,11 @@ class Qwen3_5VisionConfig(Qwen3VLVisionConfig): @strict class Qwen3_5Config(Qwen3VLConfig): r""" - num_mtp_layers (`int`, *optional*, defaults to 0): + mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. + This field is exposed as `num_mtp_layers` via `attribute_map`. + mtp_layer_types (`list[str]`, *optional*): + Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. Example: @@ -175,7 +189,19 @@ class Qwen3_5Config(Qwen3VLConfig): video_token_id: int = 248057 vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 - num_mtp_layers: int = 0 + + attribute_map = { + **Qwen3VLConfig.attribute_map, + "num_mtp_layers": "mtp_num_hidden_layers", + } + + mtp_num_hidden_layers: int = 0 + mtp_layer_types: list[str] | None = None + + def __post_init__(self, **kwargs): + super().__post_init__(**kwargs) + if self.mtp_layer_types is None: + self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers class Qwen3_5VisionRotaryEmbedding(Qwen3VLVisionRotaryEmbedding): diff --git a/tests/models/qwen3_5/test_modeling_qwen3_5.py b/tests/models/qwen3_5/test_modeling_qwen3_5.py index 36d8e210a4f0..e7389376e11a 100644 --- a/tests/models/qwen3_5/test_modeling_qwen3_5.py +++ b/tests/models/qwen3_5/test_modeling_qwen3_5.py @@ -48,6 +48,7 @@ AutoModelForImageTextToText, DynamicCache, Qwen3_5Config, + Qwen3_5DecoderLayer, Qwen3_5ForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5ForSequenceClassification, @@ -996,3 +997,22 @@ def test_model_video_generation_batch_different_videos(self): self.assertIn("Describe the video in short.", decoded_text[1]) self.assertIn("seconds>", decoded_text[1]) self.assertIn("tennis", decoded_text[1].lower()) + + +@require_torch +class Qwen3_5MtpConfigTest(unittest.TestCase): + def test_mtp_config_layer_types(self): + """MTP config must provide valid layer_types so Qwen3_5DecoderLayer does not index out of range.""" + config = Qwen3_5TextConfig(mtp_num_hidden_layers=2) + mtp_config = config.get_mtp_config() + self.assertEqual(mtp_config.num_hidden_layers, 2) + self.assertEqual(mtp_config.layer_types, ["full_attention", "full_attention"]) + layer = Qwen3_5DecoderLayer(mtp_config, 0) + self.assertEqual(layer.block_type, "full_attention") + + def test_attribute_map_num_mtp_layers(self): + """`num_mtp_layers` should alias `mtp_num_hidden_layers` for checkpoint compatibility.""" + config = Qwen3_5TextConfig(num_mtp_layers=3) + self.assertEqual(config.mtp_num_hidden_layers, 3) + self.assertEqual(config.num_mtp_layers, 3) + self.assertEqual(config.mtp_layer_types, ["full_attention", "full_attention", "full_attention"]) From ddd7a3cb53d071d4935fa65fc5fa311b14a1f56c Mon Sep 17 00:00:00 2001 From: Curnane Date: Fri, 17 Jul 2026 10:02:32 +0800 Subject: [PATCH 4/7] fix(qwen3.5): make attribute_map a plain dict --- src/transformers/modeling_layers.py | 4 +--- .../models/qwen3_5/configuration_qwen3_5.py | 10 ++-------- src/transformers/models/qwen3_5/modular_qwen3_5.py | 10 ++-------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/src/transformers/modeling_layers.py b/src/transformers/modeling_layers.py index 90186991fc2c..378e924115b5 100644 --- a/src/transformers/modeling_layers.py +++ b/src/transformers/modeling_layers.py @@ -508,9 +508,7 @@ def forward( rope_position_ids = position_ids[1:] if position_ids.shape[0] > 1 else position_ids position_embeddings = ( - self.rotary_emb(inputs_embeds, position_ids=rope_position_ids) - if self.rotary_emb is not None - else None + self.rotary_emb(inputs_embeds, position_ids=rope_position_ids) if self.rotary_emb is not None else None ) # In full generality, we may need to recompute masks for every layer due to the position offset of each layer diff --git a/src/transformers/models/qwen3_5/configuration_qwen3_5.py b/src/transformers/models/qwen3_5/configuration_qwen3_5.py index 1f40168b71a3..d103c24fb385 100644 --- a/src/transformers/models/qwen3_5/configuration_qwen3_5.py +++ b/src/transformers/models/qwen3_5/configuration_qwen3_5.py @@ -111,10 +111,7 @@ class Qwen3_5TextConfig(PreTrainedConfig): eos_token_id: int | list[int] | None = None base_config_key = "text_config" - attribute_map = { - **Qwen3NextConfig.attribute_map, - "num_mtp_layers": "mtp_num_hidden_layers", - } + attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} mtp_num_hidden_layers: int = 0 mtp_layer_types: list[str] | None = None @@ -200,10 +197,7 @@ class Qwen3_5Config(PreTrainedConfig): vision_end_token_id: int = 248054 tie_word_embeddings: bool = False - attribute_map = { - **Qwen3VLConfig.attribute_map, - "num_mtp_layers": "mtp_num_hidden_layers", - } + attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} mtp_num_hidden_layers: int = 0 mtp_layer_types: list[str] | None = None diff --git a/src/transformers/models/qwen3_5/modular_qwen3_5.py b/src/transformers/models/qwen3_5/modular_qwen3_5.py index f25794ce6555..71507e499ee4 100644 --- a/src/transformers/models/qwen3_5/modular_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modular_qwen3_5.py @@ -98,10 +98,7 @@ class Qwen3_5TextConfig(Qwen3NextConfig): model_type = "qwen3_5_text" base_config_key = "text_config" - attribute_map = { - **Qwen3NextConfig.attribute_map, - "num_mtp_layers": "mtp_num_hidden_layers", - } + attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise", @@ -190,10 +187,7 @@ class Qwen3_5Config(Qwen3VLConfig): vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 - attribute_map = { - **Qwen3VLConfig.attribute_map, - "num_mtp_layers": "mtp_num_hidden_layers", - } + attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} mtp_num_hidden_layers: int = 0 mtp_layer_types: list[str] | None = None From 83bf3c7bb6e4b43000d3c109ff6056c4b5cb2844 Mon Sep 17 00:00:00 2001 From: Curnane Date: Fri, 17 Jul 2026 10:20:11 +0800 Subject: [PATCH 5/7] revert test --- tests/models/qwen3_5/test_modeling_qwen3_5.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/tests/models/qwen3_5/test_modeling_qwen3_5.py b/tests/models/qwen3_5/test_modeling_qwen3_5.py index e7389376e11a..36d8e210a4f0 100644 --- a/tests/models/qwen3_5/test_modeling_qwen3_5.py +++ b/tests/models/qwen3_5/test_modeling_qwen3_5.py @@ -48,7 +48,6 @@ AutoModelForImageTextToText, DynamicCache, Qwen3_5Config, - Qwen3_5DecoderLayer, Qwen3_5ForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5ForSequenceClassification, @@ -997,22 +996,3 @@ def test_model_video_generation_batch_different_videos(self): self.assertIn("Describe the video in short.", decoded_text[1]) self.assertIn("seconds>", decoded_text[1]) self.assertIn("tennis", decoded_text[1].lower()) - - -@require_torch -class Qwen3_5MtpConfigTest(unittest.TestCase): - def test_mtp_config_layer_types(self): - """MTP config must provide valid layer_types so Qwen3_5DecoderLayer does not index out of range.""" - config = Qwen3_5TextConfig(mtp_num_hidden_layers=2) - mtp_config = config.get_mtp_config() - self.assertEqual(mtp_config.num_hidden_layers, 2) - self.assertEqual(mtp_config.layer_types, ["full_attention", "full_attention"]) - layer = Qwen3_5DecoderLayer(mtp_config, 0) - self.assertEqual(layer.block_type, "full_attention") - - def test_attribute_map_num_mtp_layers(self): - """`num_mtp_layers` should alias `mtp_num_hidden_layers` for checkpoint compatibility.""" - config = Qwen3_5TextConfig(num_mtp_layers=3) - self.assertEqual(config.mtp_num_hidden_layers, 3) - self.assertEqual(config.num_mtp_layers, 3) - self.assertEqual(config.mtp_layer_types, ["full_attention", "full_attention", "full_attention"]) From 2568d4842cf86d0b18b8164f4ce212275eccdd93 Mon Sep 17 00:00:00 2001 From: Curnane Date: Sun, 19 Jul 2026 11:54:02 +0800 Subject: [PATCH 6/7] fix(qwen3.5): make mtp_layer_types a computed property --- .../models/qwen3_5/configuration_qwen3_5.py | 24 ++++++++++-------- .../models/qwen3_5/modular_qwen3_5.py | 25 ++++++++++--------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/transformers/models/qwen3_5/configuration_qwen3_5.py b/src/transformers/models/qwen3_5/configuration_qwen3_5.py index d103c24fb385..a89330333426 100644 --- a/src/transformers/models/qwen3_5/configuration_qwen3_5.py +++ b/src/transformers/models/qwen3_5/configuration_qwen3_5.py @@ -41,8 +41,6 @@ class Qwen3_5TextConfig(PreTrainedConfig): mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. This field is exposed as `num_mtp_layers` via `attribute_map`. - mtp_layer_types (`list[str]`, *optional*): - Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -114,7 +112,6 @@ class Qwen3_5TextConfig(PreTrainedConfig): attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} mtp_num_hidden_layers: int = 0 - mtp_layer_types: list[str] | None = None def __post_init__(self, **kwargs): kwargs.setdefault("partial_rotary_factor", 0.25) # assign default for BC @@ -128,8 +125,13 @@ def __post_init__(self, **kwargs): self.layer_types = remap_legacy_layer_types(self.layer_types) super().__post_init__(**kwargs) - if self.mtp_layer_types is None: - self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers + + # MTP layers always use full attention + @property + def mtp_layer_types(self): + if self.num_mtp_layers is not None: + return ["full_attention"] * self.num_mtp_layers + return None @auto_docstring(checkpoint="Qwen/Qwen3.5-27B") @@ -166,8 +168,6 @@ class Qwen3_5Config(PreTrainedConfig): mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. This field is exposed as `num_mtp_layers` via `attribute_map`. - mtp_layer_types (`list[str]`, *optional*): - Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. Example: @@ -200,7 +200,6 @@ class Qwen3_5Config(PreTrainedConfig): attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} mtp_num_hidden_layers: int = 0 - mtp_layer_types: list[str] | None = None def __post_init__(self, **kwargs): if isinstance(self.vision_config, dict): @@ -217,8 +216,13 @@ def __post_init__(self, **kwargs): self.text_config = self.sub_configs["text_config"]() super().__post_init__(**kwargs) - if self.mtp_layer_types is None: - self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers + + # MTP layers always use full attention + @property + def mtp_layer_types(self): + if self.num_mtp_layers is not None: + return ["full_attention"] * self.num_mtp_layers + return None __all__ = ["Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5VisionConfig"] diff --git a/src/transformers/models/qwen3_5/modular_qwen3_5.py b/src/transformers/models/qwen3_5/modular_qwen3_5.py index 71507e499ee4..09bc04d321f5 100644 --- a/src/transformers/models/qwen3_5/modular_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modular_qwen3_5.py @@ -78,8 +78,6 @@ class Qwen3_5TextConfig(Qwen3NextConfig): mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. This field is exposed as `num_mtp_layers` via `attribute_map`. - mtp_layer_types (`list[str]`, *optional*): - Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. ```python >>> from transformers import Qwen3_5TextModel, Qwen3_5TextConfig @@ -125,7 +123,6 @@ class Qwen3_5TextConfig(Qwen3NextConfig): num_hidden_layers: int = 32 num_key_value_heads: int = 4 mtp_num_hidden_layers: int = 0 - mtp_layer_types: list[str] | None = None decoder_sparse_step = AttributeError() norm_topk_prob = AttributeError() @@ -139,10 +136,15 @@ class Qwen3_5TextConfig(Qwen3NextConfig): def __post_init__(self, **kwargs): super().__post_init__(**kwargs) - if self.mtp_layer_types is None: - self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers del self.mlp_only_layers + # MTP layers always use full attention + @property + def mtp_layer_types(self): + if self.num_mtp_layers is not None: + return ["full_attention"] * self.num_mtp_layers + return None + @auto_docstring(checkpoint="Qwen/Qwen3.5-27B") @strict @@ -164,8 +166,6 @@ class Qwen3_5Config(Qwen3VLConfig): mtp_num_hidden_layers (`int`, *optional*, defaults to 0): Number of Multi-Token Prediction (MTP) layers. When set to 0, MTP is disabled. This field is exposed as `num_mtp_layers` via `attribute_map`. - mtp_layer_types (`list[str]`, *optional*): - Layer types for the MTP layers. Defaults to `["full_attention"] * mtp_num_hidden_layers`. Example: @@ -190,12 +190,13 @@ class Qwen3_5Config(Qwen3VLConfig): attribute_map = {"num_mtp_layers": "mtp_num_hidden_layers"} mtp_num_hidden_layers: int = 0 - mtp_layer_types: list[str] | None = None - def __post_init__(self, **kwargs): - super().__post_init__(**kwargs) - if self.mtp_layer_types is None: - self.mtp_layer_types = ["full_attention"] * self.mtp_num_hidden_layers + # MTP layers always use full attention + @property + def mtp_layer_types(self): + if self.num_mtp_layers is not None: + return ["full_attention"] * self.num_mtp_layers + return None class Qwen3_5VisionRotaryEmbedding(Qwen3VLVisionRotaryEmbedding): From 8277ee005627f9da42b3347da81f63a0a18b7caf Mon Sep 17 00:00:00 2001 From: Curnane Date: Sun, 19 Jul 2026 16:44:04 +0800 Subject: [PATCH 7/7] test: skip MTP generation for stateful models --- tests/generation/test_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index e494c63d1c60..d4dadbb342a8 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -2372,6 +2372,11 @@ def test_generate_with_mtp(self): ): self.skipTest("No MTP keys registered") + # Assisted generation (use_mtp) is blocked for stateful models, whose recurrent state + # cannot be rolled back during candidate verification + if getattr(model_class, "_is_stateful", False): + self.skipTest("MTP generation is not supported for stateful models") + config.get_text_config().num_mtp_layers = 1 model = model_class(config).to(torch_device).eval() mtp_model = MtpModel(model, num_mtp_layers=1)