Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/transformers/generation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2706,9 +2706,11 @@ def generate(
# We can drop the mask altogether if it's all 1s, i.e. no padding, to make downstream attention mask creation and inference
# faster (we will never have padding). Note that we cannot drop it earlier, as position_ids creation absolutely needs to check
# the mask even if it's only 1s, in case we restart from an existing cache and only new sequence input_ids
# Keep the mask for eager attention to preserve model-side mask construction.
if (
not self.config.is_encoder_decoder
and accepts_attention_mask
and self.config.get_text_config()._attn_implementation != "eager"

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should do that @Cyrilvallez ?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm I know flex attn was never actually used for inference but was it considered at some point? 😢

Comment thread
SunMarc marked this conversation as resolved.
Resolved
and (model_kwargs["attention_mask"] == 1).all()
):
# Record the length to slice correctly in `_prefill` if restarting from existing Cache
Expand Down
1 change: 1 addition & 0 deletions src/transformers/models/imagegpt/modeling_imagegpt.py
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ def forward(
config=self.config,
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
)

if encoder_attention_mask is not None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -622,15 +622,13 @@ def forward(
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
position_ids = position_ids.unsqueeze(0)

causal_mask = None
if attention_mask is not None:
causal_mask = create_causal_mask(
config=self.config,
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
position_ids=position_ids,
)
causal_mask = create_causal_mask(
config=self.config,
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
position_ids=position_ids,
)

# embed positions
hidden_states = inputs_embeds
Expand Down
16 changes: 7 additions & 9 deletions src/transformers/models/moshi/modeling_moshi.py
Original file line number Diff line number Diff line change
Expand Up @@ -852,15 +852,13 @@ def forward(
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
position_ids = position_ids.unsqueeze(0)

causal_mask = None
if attention_mask is not None:
causal_mask = create_causal_mask(
config=self.config,
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
position_ids=position_ids,
)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

otherwise the fix is this one but i think the issue with eager might happen with other models as well

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be fine, no reason to do this prev pattern

@SunMarc SunMarc Sep 17, 2026 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

okay then we can probably drop the eager check no ?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The eager check was only needed for this?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Imo, we should never do

if mask is none: create_mask

that portion should properly handle the skips and creation and is an anti pattern to the create fn

causal_mask = create_causal_mask(
config=self.config,
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
past_key_values=past_key_values,
position_ids=position_ids,
)

# embed positions
hidden_states = inputs_embeds
Expand Down
138 changes: 114 additions & 24 deletions src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3086,27 +3086,17 @@ def forward(
generation_step = -1
residual_codes = None
if position_ids is None:
past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length()
if past_key_values_length == 0 or self.rope_deltas is None:
delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1)
position_ids, rope_deltas = self.get_rope_index(
talker_input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video,
audio_feature_lengths,
video_second_per_grid,
)
rope_deltas = rope_deltas - delta0
self.rope_deltas = rope_deltas
else:
batch_size, seq_length = input_ids.shape
delta = (past_key_values_length + self.rope_deltas).to(input_ids.device)
position_ids = torch.arange(seq_length, device=input_ids.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
position_ids = position_ids.add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
position_ids = self.compute_3d_position_ids(
inputs_embeds,
talker_input_ids=talker_input_ids,
attention_mask=attention_mask,
past_key_values=past_key_values,
image_grid_thw=image_grid_thw,
video_grid_thw=video_grid_thw,
use_audio_in_video=use_audio_in_video,
audio_feature_lengths=audio_feature_lengths,
video_second_per_grid=video_second_per_grid,
)

outputs: MoeModelOutputWithPast = self.model(
input_ids=None,
Expand Down Expand Up @@ -3187,6 +3177,109 @@ def get_llm_pos_ids_for_vision(
def get_input_embeddings(self):
return self.model.get_input_embeddings()

def compute_3d_position_ids(
self,
inputs_tensor: torch.Tensor,
talker_input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
past_key_values: Cache | None = None,
image_grid_thw: torch.Tensor | None = None,
video_grid_thw: torch.Tensor | None = None,
use_audio_in_video: bool | None = None,
audio_feature_lengths: torch.LongTensor | None = None,
video_second_per_grid: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size, seq_length = inputs_tensor.shape[:2]
past_length = 0 if past_key_values is None else past_key_values.get_seq_length()
has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw))
if past_length == 0 or self.rope_deltas is None:
if attention_mask is None:
attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device)
if talker_input_ids is not None and has_multimodal_data:
position_ids, rope_deltas = self.get_rope_index(
talker_input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video or False,
audio_feature_lengths,
video_second_per_grid,
)
else:
position_ids = attention_mask.long().cumsum(-1) - 1
position_ids = position_ids.masked_fill(attention_mask == 0, 0)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
rope_deltas = torch.zeros(batch_size, 1, dtype=torch.long, device=inputs_tensor.device)
self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1)
else:
delta = (past_length + self.rope_deltas).to(inputs_tensor.device)
position_ids = torch.arange(seq_length, device=inputs_tensor.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1).add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
return position_ids

def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs):
# Overwritten -- requires 3D position ids

text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs)
attention_mask = model_kwargs.get("attention_mask")
padding_length = (1 - attention_mask).sum(dim=-1, keepdim=True) if attention_mask is not None else 0

# Early exit in case we are continuing generation from past kv
past_length = 0
if (cache := model_kwargs.get("past_key_values")) is not None:
past_length = cache.get_seq_length()
if past_length != 0 and self.rope_deltas is not None:
# Talker stores deltas relative to the physical cache length, which includes padding.
vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas + padding_length
position_ids = torch.cat([text_positions[None, ...], vision_positions], dim=0)
return position_ids

# Otherwise compute 3d position ids for audio/vision tokens and concat with text position ids
if "talker_input_ids" in model_kwargs and model_kwargs["talker_input_ids"] is not None:
inputs_tensor = model_kwargs["talker_input_ids"]

is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long]
has_multimodal_data = any(
grid is not None and grid.numel() > 0
for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw"))
)
if is_input_ids and attention_mask is not None and has_multimodal_data:
vision_positions, rope_deltas = self.get_rope_index(
inputs_tensor,
image_grid_thw=model_kwargs.get("image_grid_thw"),
video_grid_thw=model_kwargs.get("video_grid_thw"),
attention_mask=attention_mask,
use_audio_in_video=model_kwargs.get("use_audio_in_video") or False,
audio_seqlens=model_kwargs.get("audio_feature_lengths"),
second_per_grids=model_kwargs.get("video_second_per_grid"),
)
self.rope_deltas = rope_deltas - padding_length
else:
vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1)
self.rope_deltas = (
torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device) - padding_length
)

# Concatenate "text + vision" positions into [4, bs, seq-len]
text_positions = text_positions[None, ...]
position_ids = torch.cat([text_positions, vision_positions], dim=0)

return position_ids

def _expand_inputs_for_generation(self, expand_size=1, is_encoder_decoder=False, input_ids=None, **model_kwargs):
position_ids = model_kwargs.pop("position_ids", None)
input_ids, model_kwargs = super()._expand_inputs_for_generation(
expand_size=expand_size,
is_encoder_decoder=is_encoder_decoder,
input_ids=input_ids,
**model_kwargs,
)
if position_ids is not None:
batch_dim = 1 if position_ids.ndim == 3 else 0
model_kwargs["position_ids"] = position_ids.repeat_interleave(expand_size, dim=batch_dim)
return input_ids, model_kwargs

def _update_model_kwargs_for_generation(self, outputs, model_kwargs, is_encoder_decoder=False, num_new_tokens=1):
model_kwargs = super()._update_model_kwargs_for_generation(
outputs, model_kwargs, is_encoder_decoder, num_new_tokens
Expand Down Expand Up @@ -3222,9 +3315,6 @@ def prepare_inputs_for_generation(
**kwargs,
)

# Qwen3-Omni will prepare position ids in forward with deltas
inputs["position_ids"] = None

# TODO(raushan, gante): Refactor this part to a utility function
if not is_first_iteration and kwargs.get("use_cache", True):
input_ids = input_ids[:, -1:]
Expand Down
138 changes: 114 additions & 24 deletions src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -1737,6 +1737,109 @@ def get_llm_pos_ids_for_vision(
def get_input_embeddings(self):
return self.model.get_input_embeddings()

def compute_3d_position_ids(
self,
inputs_tensor: torch.Tensor,
talker_input_ids: torch.LongTensor | None = None,
attention_mask: torch.Tensor | None = None,
past_key_values: Cache | None = None,
image_grid_thw: torch.Tensor | None = None,
video_grid_thw: torch.Tensor | None = None,
use_audio_in_video: bool | None = None,
audio_feature_lengths: torch.LongTensor | None = None,
video_second_per_grid: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size, seq_length = inputs_tensor.shape[:2]
past_length = 0 if past_key_values is None else past_key_values.get_seq_length()
has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw))

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or audio_seqlens is not None :)

if past_length == 0 or self.rope_deltas is None:
if attention_mask is None:
attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device)
if talker_input_ids is not None and has_multimodal_data:
position_ids, rope_deltas = self.get_rope_index(
talker_input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video or False,
audio_feature_lengths,
video_second_per_grid,
)
else:
position_ids = attention_mask.long().cumsum(-1) - 1
position_ids = position_ids.masked_fill(attention_mask == 0, 0)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
rope_deltas = torch.zeros(batch_size, 1, dtype=torch.long, device=inputs_tensor.device)
self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1)
else:
delta = (past_length + self.rope_deltas).to(inputs_tensor.device)
position_ids = torch.arange(seq_length, device=inputs_tensor.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1).add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
return position_ids

def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs):
# Overwritten -- requires 3D position ids

Comment on lines +1783 to +1785

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh i love qwen!

text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs)
attention_mask = model_kwargs.get("attention_mask")
padding_length = (1 - attention_mask).sum(dim=-1, keepdim=True) if attention_mask is not None else 0

# Early exit in case we are continuing generation from past kv
past_length = 0
if (cache := model_kwargs.get("past_key_values")) is not None:
past_length = cache.get_seq_length()
if past_length != 0 and self.rope_deltas is not None:
# Talker stores deltas relative to the physical cache length, which includes padding.
vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas + padding_length
position_ids = torch.cat([text_positions[None, ...], vision_positions], dim=0)
return position_ids

# Otherwise compute 3d position ids for audio/vision tokens and concat with text position ids
if "talker_input_ids" in model_kwargs and model_kwargs["talker_input_ids"] is not None:
inputs_tensor = model_kwargs["talker_input_ids"]

is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long]
has_multimodal_data = any(
grid is not None and grid.numel() > 0
for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw"))
)
if is_input_ids and attention_mask is not None and has_multimodal_data:
vision_positions, rope_deltas = self.get_rope_index(
inputs_tensor,
image_grid_thw=model_kwargs.get("image_grid_thw"),
video_grid_thw=model_kwargs.get("video_grid_thw"),
attention_mask=attention_mask,
use_audio_in_video=model_kwargs.get("use_audio_in_video") or False,
audio_seqlens=model_kwargs.get("audio_feature_lengths"),
second_per_grids=model_kwargs.get("video_second_per_grid"),
)
self.rope_deltas = rope_deltas - padding_length
else:
vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1)
self.rope_deltas = (
torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device) - padding_length
)

# Concatenate "text + vision" positions into [4, bs, seq-len]
text_positions = text_positions[None, ...]
position_ids = torch.cat([text_positions, vision_positions], dim=0)

return position_ids

def _expand_inputs_for_generation(self, expand_size=1, is_encoder_decoder=False, input_ids=None, **model_kwargs):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ig this comes from other qwen vls? at least the pattern

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

would like Raushan's opinion on this

position_ids = model_kwargs.pop("position_ids", None)
input_ids, model_kwargs = super()._expand_inputs_for_generation(
expand_size=expand_size,
is_encoder_decoder=is_encoder_decoder,
input_ids=input_ids,
**model_kwargs,
)
if position_ids is not None:
batch_dim = 1 if position_ids.ndim == 3 else 0
model_kwargs["position_ids"] = position_ids.repeat_interleave(expand_size, dim=batch_dim)
return input_ids, model_kwargs
Comment on lines +1832 to +1843

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this defi doesn't work well for images/videos 😆

old hidden bug, but let it be - this model is almost never used atm, and we'll remove all overriden expansion soon 🤞🏻

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

okay XD


def forward(
self,
input_ids=None,
Expand Down Expand Up @@ -1782,27 +1885,17 @@ def forward(
generation_step = -1
residual_codes = None
if position_ids is None:
past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length()
if past_key_values_length == 0 or self.rope_deltas is None:
delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1)
position_ids, rope_deltas = self.get_rope_index(
talker_input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video,
audio_feature_lengths,
video_second_per_grid,
)
rope_deltas = rope_deltas - delta0
self.rope_deltas = rope_deltas
else:
batch_size, seq_length = input_ids.shape
delta = (past_key_values_length + self.rope_deltas).to(input_ids.device)
position_ids = torch.arange(seq_length, device=input_ids.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
position_ids = position_ids.add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
position_ids = self.compute_3d_position_ids(
inputs_embeds,
talker_input_ids=talker_input_ids,
attention_mask=attention_mask,
past_key_values=past_key_values,
image_grid_thw=image_grid_thw,
video_grid_thw=video_grid_thw,
use_audio_in_video=use_audio_in_video,
audio_feature_lengths=audio_feature_lengths,
video_second_per_grid=video_second_per_grid,
)

outputs: MoeModelOutputWithPast = self.model(
input_ids=None,
Expand Down Expand Up @@ -1880,9 +1973,6 @@ def prepare_inputs_for_generation(
**kwargs,
)

# Qwen3-Omni will prepare position ids in forward with deltas
inputs["position_ids"] = None

# TODO(raushan, gante): Refactor this part to a utility function
if not is_first_iteration and kwargs.get("use_cache", True):
input_ids = input_ids[:, -1:]
Expand Down
Loading