From 6d3618cf21ee68abad1358ddbd955b1a18bcaa7c Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 17 Sep 2026 14:51:21 +0200 Subject: [PATCH 1/6] fix tests --- .../models/imagegpt/modeling_imagegpt.py | 1 + src/transformers/models/moshi/modeling_moshi.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index b716332507b0..9fc32ed88971 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -521,6 +521,7 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, + past_key_values=past_key_values, ) if encoder_attention_mask is not None: diff --git a/src/transformers/models/moshi/modeling_moshi.py b/src/transformers/models/moshi/modeling_moshi.py index 3d784cb83f78..ae026cc446ae 100644 --- a/src/transformers/models/moshi/modeling_moshi.py +++ b/src/transformers/models/moshi/modeling_moshi.py @@ -852,15 +852,13 @@ def forward( position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens position_ids = position_ids.unsqueeze(0) - causal_mask = None - if attention_mask is not None: - causal_mask = create_causal_mask( - config=self.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - ) + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) # embed positions hidden_states = inputs_embeds From 2f91c6ae28694ee3de533dd845e3dac66f4652a5 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 17 Sep 2026 16:53:09 +0200 Subject: [PATCH 2/6] fix ! --- src/transformers/generation/utils.py | 2 + .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 127 ++++++++++++++---- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 127 ++++++++++++++---- 3 files changed, 208 insertions(+), 48 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 90a1574349cc..de0d9027e537 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2706,9 +2706,11 @@ def generate( # We can drop the mask altogether if it's all 1s, i.e. no padding, to make downstream attention mask creation and inference # faster (we will never have padding). Note that we cannot drop it earlier, as position_ids creation absolutely needs to check # the mask even if it's only 1s, in case we restart from an existing cache and only new sequence input_ids + # Keep the mask for eager attention to preserve model-side mask construction. if ( not self.config.is_encoder_decoder and accepts_attention_mask + and self.config.get_text_config()._attn_implementation != "eager" and (model_kwargs["attention_mask"] == 1).all() ): # Record the length to slice correctly in `_prefill` if restarting from existing Cache diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 07a81845fbf8..9a4046d12c85 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -3086,27 +3086,17 @@ def forward( generation_step = -1 residual_codes = None if position_ids is None: - past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length() - if past_key_values_length == 0 or self.rope_deltas is None: - delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1) - position_ids, rope_deltas = self.get_rope_index( - talker_input_ids, - image_grid_thw, - video_grid_thw, - attention_mask, - use_audio_in_video, - audio_feature_lengths, - video_second_per_grid, - ) - rope_deltas = rope_deltas - delta0 - self.rope_deltas = rope_deltas - else: - batch_size, seq_length = input_ids.shape - delta = (past_key_values_length + self.rope_deltas).to(input_ids.device) - position_ids = torch.arange(seq_length, device=input_ids.device) - position_ids = position_ids.view(1, -1).expand(batch_size, -1) - position_ids = position_ids.add(delta) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + position_ids = self.compute_3d_position_ids( + inputs_embeds, + talker_input_ids=talker_input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + use_audio_in_video=use_audio_in_video, + audio_feature_lengths=audio_feature_lengths, + video_second_per_grid=video_second_per_grid, + ) outputs: MoeModelOutputWithPast = self.model( input_ids=None, @@ -3187,6 +3177,98 @@ def get_llm_pos_ids_for_vision( def get_input_embeddings(self): return self.model.get_input_embeddings() + def compute_3d_position_ids( + self, + inputs_tensor: torch.Tensor, + talker_input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + image_grid_thw: torch.Tensor | None = None, + video_grid_thw: torch.Tensor | None = None, + use_audio_in_video: bool | None = None, + audio_feature_lengths: torch.LongTensor | None = None, + video_second_per_grid: torch.Tensor | None = None, + ) -> torch.Tensor: + batch_size, seq_length = inputs_tensor.shape[:2] + past_length = 0 if past_key_values is None else past_key_values.get_seq_length() + if past_length == 0 or self.rope_deltas is None: + if attention_mask is None: + attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) + position_ids, rope_deltas = self.get_rope_index( + talker_input_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + use_audio_in_video or False, + audio_feature_lengths, + video_second_per_grid, + ) + self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1) + else: + delta = (past_length + self.rope_deltas).to(inputs_tensor.device) + position_ids = torch.arange(seq_length, device=inputs_tensor.device) + position_ids = position_ids.view(1, -1).expand(batch_size, -1).add(delta) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + return position_ids + + def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): + # Overwritten -- requires 3D position ids + + text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs) + attention_mask = model_kwargs.get("attention_mask") + padding_length = (1 - attention_mask).sum(dim=-1, keepdim=True) if attention_mask is not None else 0 + + # Early exit in case we are continuing generation from past kv + past_length = 0 + if (cache := model_kwargs.get("past_key_values")) is not None: + past_length = cache.get_seq_length() + if past_length != 0 and self.rope_deltas is not None: + # Talker stores deltas relative to the physical cache length, which includes padding. + vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas + padding_length + position_ids = torch.cat([text_positions[None, ...], vision_positions], dim=0) + return position_ids + + # Otherwise compute 3d position ids for audio/vision tokens and concat with text position ids + if "talker_input_ids" in model_kwargs and model_kwargs["talker_input_ids"] is not None: + inputs_tensor = model_kwargs["talker_input_ids"] + + is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] + if is_input_ids and attention_mask is not None: + vision_positions, rope_deltas = self.get_rope_index( + inputs_tensor, + image_grid_thw=model_kwargs.get("image_grid_thw"), + video_grid_thw=model_kwargs.get("video_grid_thw"), + attention_mask=attention_mask, + use_audio_in_video=model_kwargs.get("use_audio_in_video") or False, + audio_seqlens=model_kwargs.get("audio_feature_lengths"), + second_per_grids=model_kwargs.get("video_second_per_grid"), + ) + self.rope_deltas = rope_deltas - padding_length + else: + vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas = ( + torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device) - padding_length + ) + + # Concatenate "text + vision" positions into [4, bs, seq-len] + text_positions = text_positions[None, ...] + position_ids = torch.cat([text_positions, vision_positions], dim=0) + + return position_ids + + def _expand_inputs_for_generation(self, expand_size=1, is_encoder_decoder=False, input_ids=None, **model_kwargs): + position_ids = model_kwargs.pop("position_ids", None) + input_ids, model_kwargs = super()._expand_inputs_for_generation( + expand_size=expand_size, + is_encoder_decoder=is_encoder_decoder, + input_ids=input_ids, + **model_kwargs, + ) + if position_ids is not None: + batch_dim = 1 if position_ids.ndim == 3 else 0 + model_kwargs["position_ids"] = position_ids.repeat_interleave(expand_size, dim=batch_dim) + return input_ids, model_kwargs + def _update_model_kwargs_for_generation(self, outputs, model_kwargs, is_encoder_decoder=False, num_new_tokens=1): model_kwargs = super()._update_model_kwargs_for_generation( outputs, model_kwargs, is_encoder_decoder, num_new_tokens @@ -3222,9 +3304,6 @@ def prepare_inputs_for_generation( **kwargs, ) - # Qwen3-Omni will prepare position ids in forward with deltas - inputs["position_ids"] = None - # TODO(raushan, gante): Refactor this part to a utility function if not is_first_iteration and kwargs.get("use_cache", True): input_ids = input_ids[:, -1:] diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 7cde9ddff8fd..1dbb5876bde5 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1737,6 +1737,98 @@ def get_llm_pos_ids_for_vision( def get_input_embeddings(self): return self.model.get_input_embeddings() + def compute_3d_position_ids( + self, + inputs_tensor: torch.Tensor, + talker_input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + past_key_values: Cache | None = None, + image_grid_thw: torch.Tensor | None = None, + video_grid_thw: torch.Tensor | None = None, + use_audio_in_video: bool | None = None, + audio_feature_lengths: torch.LongTensor | None = None, + video_second_per_grid: torch.Tensor | None = None, + ) -> torch.Tensor: + batch_size, seq_length = inputs_tensor.shape[:2] + past_length = 0 if past_key_values is None else past_key_values.get_seq_length() + if past_length == 0 or self.rope_deltas is None: + if attention_mask is None: + attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) + position_ids, rope_deltas = self.get_rope_index( + talker_input_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + use_audio_in_video or False, + audio_feature_lengths, + video_second_per_grid, + ) + self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1) + else: + delta = (past_length + self.rope_deltas).to(inputs_tensor.device) + position_ids = torch.arange(seq_length, device=inputs_tensor.device) + position_ids = position_ids.view(1, -1).expand(batch_size, -1).add(delta) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + return position_ids + + def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): + # Overwritten -- requires 3D position ids + + text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs) + attention_mask = model_kwargs.get("attention_mask") + padding_length = (1 - attention_mask).sum(dim=-1, keepdim=True) if attention_mask is not None else 0 + + # Early exit in case we are continuing generation from past kv + past_length = 0 + if (cache := model_kwargs.get("past_key_values")) is not None: + past_length = cache.get_seq_length() + if past_length != 0 and self.rope_deltas is not None: + # Talker stores deltas relative to the physical cache length, which includes padding. + vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas + padding_length + position_ids = torch.cat([text_positions[None, ...], vision_positions], dim=0) + return position_ids + + # Otherwise compute 3d position ids for audio/vision tokens and concat with text position ids + if "talker_input_ids" in model_kwargs and model_kwargs["talker_input_ids"] is not None: + inputs_tensor = model_kwargs["talker_input_ids"] + + is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] + if is_input_ids and attention_mask is not None: + vision_positions, rope_deltas = self.get_rope_index( + inputs_tensor, + image_grid_thw=model_kwargs.get("image_grid_thw"), + video_grid_thw=model_kwargs.get("video_grid_thw"), + attention_mask=attention_mask, + use_audio_in_video=model_kwargs.get("use_audio_in_video") or False, + audio_seqlens=model_kwargs.get("audio_feature_lengths"), + second_per_grids=model_kwargs.get("video_second_per_grid"), + ) + self.rope_deltas = rope_deltas - padding_length + else: + vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas = ( + torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device) - padding_length + ) + + # Concatenate "text + vision" positions into [4, bs, seq-len] + text_positions = text_positions[None, ...] + position_ids = torch.cat([text_positions, vision_positions], dim=0) + + return position_ids + + def _expand_inputs_for_generation(self, expand_size=1, is_encoder_decoder=False, input_ids=None, **model_kwargs): + position_ids = model_kwargs.pop("position_ids", None) + input_ids, model_kwargs = super()._expand_inputs_for_generation( + expand_size=expand_size, + is_encoder_decoder=is_encoder_decoder, + input_ids=input_ids, + **model_kwargs, + ) + if position_ids is not None: + batch_dim = 1 if position_ids.ndim == 3 else 0 + model_kwargs["position_ids"] = position_ids.repeat_interleave(expand_size, dim=batch_dim) + return input_ids, model_kwargs + def forward( self, input_ids=None, @@ -1782,27 +1874,17 @@ def forward( generation_step = -1 residual_codes = None if position_ids is None: - past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length() - if past_key_values_length == 0 or self.rope_deltas is None: - delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1) - position_ids, rope_deltas = self.get_rope_index( - talker_input_ids, - image_grid_thw, - video_grid_thw, - attention_mask, - use_audio_in_video, - audio_feature_lengths, - video_second_per_grid, - ) - rope_deltas = rope_deltas - delta0 - self.rope_deltas = rope_deltas - else: - batch_size, seq_length = input_ids.shape - delta = (past_key_values_length + self.rope_deltas).to(input_ids.device) - position_ids = torch.arange(seq_length, device=input_ids.device) - position_ids = position_ids.view(1, -1).expand(batch_size, -1) - position_ids = position_ids.add(delta) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + position_ids = self.compute_3d_position_ids( + inputs_embeds, + talker_input_ids=talker_input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + use_audio_in_video=use_audio_in_video, + audio_feature_lengths=audio_feature_lengths, + video_second_per_grid=video_second_per_grid, + ) outputs: MoeModelOutputWithPast = self.model( input_ids=None, @@ -1880,9 +1962,6 @@ def prepare_inputs_for_generation( **kwargs, ) - # Qwen3-Omni will prepare position ids in forward with deltas - inputs["position_ids"] = None - # TODO(raushan, gante): Refactor this part to a utility function if not is_first_iteration and kwargs.get("use_cache", True): input_ids = input_ids[:, -1:] From e7aba70fb93040a9b7f1e1cd859686d952a0c81b Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 17 Sep 2026 17:36:40 +0200 Subject: [PATCH 3/6] updated ! --- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 31 +++++++++++++------ .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 31 +++++++++++++------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 9a4046d12c85..8a043a6810d9 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -3191,18 +3191,25 @@ def compute_3d_position_ids( ) -> torch.Tensor: batch_size, seq_length = inputs_tensor.shape[:2] past_length = 0 if past_key_values is None else past_key_values.get_seq_length() + has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw)) if past_length == 0 or self.rope_deltas is None: if attention_mask is None: attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) - position_ids, rope_deltas = self.get_rope_index( - talker_input_ids, - image_grid_thw, - video_grid_thw, - attention_mask, - use_audio_in_video or False, - audio_feature_lengths, - video_second_per_grid, - ) + if talker_input_ids is not None and has_multimodal_data: + position_ids, rope_deltas = self.get_rope_index( + talker_input_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + use_audio_in_video or False, + audio_feature_lengths, + video_second_per_grid, + ) + else: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids = position_ids.masked_fill(attention_mask == 0, 0) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + rope_deltas = torch.zeros(batch_size, 1, dtype=torch.long, device=inputs_tensor.device) self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1) else: delta = (past_length + self.rope_deltas).to(inputs_tensor.device) @@ -3233,7 +3240,11 @@ def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): inputs_tensor = model_kwargs["talker_input_ids"] is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] - if is_input_ids and attention_mask is not None: + has_multimodal_data = any( + grid is not None and grid.numel() > 0 + for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw")) + ) + if is_input_ids and attention_mask is not None and has_multimodal_data: vision_positions, rope_deltas = self.get_rope_index( inputs_tensor, image_grid_thw=model_kwargs.get("image_grid_thw"), diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 1dbb5876bde5..2c3ec7e42b1f 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1751,18 +1751,25 @@ def compute_3d_position_ids( ) -> torch.Tensor: batch_size, seq_length = inputs_tensor.shape[:2] past_length = 0 if past_key_values is None else past_key_values.get_seq_length() + has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw)) if past_length == 0 or self.rope_deltas is None: if attention_mask is None: attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) - position_ids, rope_deltas = self.get_rope_index( - talker_input_ids, - image_grid_thw, - video_grid_thw, - attention_mask, - use_audio_in_video or False, - audio_feature_lengths, - video_second_per_grid, - ) + if talker_input_ids is not None and has_multimodal_data: + position_ids, rope_deltas = self.get_rope_index( + talker_input_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + use_audio_in_video or False, + audio_feature_lengths, + video_second_per_grid, + ) + else: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids = position_ids.masked_fill(attention_mask == 0, 0) + position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) + rope_deltas = torch.zeros(batch_size, 1, dtype=torch.long, device=inputs_tensor.device) self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1) else: delta = (past_length + self.rope_deltas).to(inputs_tensor.device) @@ -1793,7 +1800,11 @@ def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): inputs_tensor = model_kwargs["talker_input_ids"] is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] - if is_input_ids and attention_mask is not None: + has_multimodal_data = any( + grid is not None and grid.numel() > 0 + for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw")) + ) + if is_input_ids and attention_mask is not None and has_multimodal_data: vision_positions, rope_deltas = self.get_rope_index( inputs_tensor, image_grid_thw=model_kwargs.get("image_grid_thw"), From 868c6d5c6a1b53e74d64dcb8da016f77551a78db Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 17 Sep 2026 17:42:52 +0200 Subject: [PATCH 4/6] up --- .../modeling_kyutai_speech_to_text.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py index be53792e7dd3..079ee82c2226 100644 --- a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py +++ b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py @@ -622,15 +622,13 @@ def forward( position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens position_ids = position_ids.unsqueeze(0) - causal_mask = None - if attention_mask is not None: - causal_mask = create_causal_mask( - config=self.config, - inputs_embeds=inputs_embeds, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - ) + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) # embed positions hidden_states = inputs_embeds From 0f77400c041d8b4cb3d163b8b712a5c024e2207f Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 17 Sep 2026 18:30:49 +0200 Subject: [PATCH 5/6] audioo --- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 6 ++++-- .../models/qwen3_omni_moe/modular_qwen3_omni_moe.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 8a043a6810d9..48ceafa5d7ff 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -3191,7 +3191,9 @@ def compute_3d_position_ids( ) -> torch.Tensor: batch_size, seq_length = inputs_tensor.shape[:2] past_length = 0 if past_key_values is None else past_key_values.get_seq_length() - has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw)) + has_multimodal_data = audio_feature_lengths is not None or any( + grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw) + ) if past_length == 0 or self.rope_deltas is None: if attention_mask is None: attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) @@ -3240,7 +3242,7 @@ def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): inputs_tensor = model_kwargs["talker_input_ids"] is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] - has_multimodal_data = any( + has_multimodal_data = model_kwargs.get("audio_feature_lengths") is not None or any( grid is not None and grid.numel() > 0 for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw")) ) diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 2c3ec7e42b1f..9c3c23e1966c 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1751,7 +1751,9 @@ def compute_3d_position_ids( ) -> torch.Tensor: batch_size, seq_length = inputs_tensor.shape[:2] past_length = 0 if past_key_values is None else past_key_values.get_seq_length() - has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw)) + has_multimodal_data = audio_feature_lengths is not None or any( + grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw) + ) if past_length == 0 or self.rope_deltas is None: if attention_mask is None: attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) @@ -1800,7 +1802,7 @@ def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): inputs_tensor = model_kwargs["talker_input_ids"] is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] - has_multimodal_data = any( + has_multimodal_data = model_kwargs.get("audio_feature_lengths") is not None or any( grid is not None and grid.numel() > 0 for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw")) ) From f042b2ed0bd430ba830d387e46a944f3bbfef16d Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Fri, 18 Sep 2026 15:53:09 +0200 Subject: [PATCH 6/6] remove unneeded fix --- src/transformers/generation/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index de0d9027e537..90a1574349cc 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2706,11 +2706,9 @@ def generate( # We can drop the mask altogether if it's all 1s, i.e. no padding, to make downstream attention mask creation and inference # faster (we will never have padding). Note that we cannot drop it earlier, as position_ids creation absolutely needs to check # the mask even if it's only 1s, in case we restart from an existing cache and only new sequence input_ids - # Keep the mask for eager attention to preserve model-side mask construction. if ( not self.config.is_encoder_decoder and accepts_attention_mask - and self.config.get_text_config()._attn_implementation != "eager" and (model_kwargs["attention_mask"] == 1).all() ): # Record the length to slice correctly in `_prefill` if restarting from existing Cache