Repository navigation
Fix tests due to dropping attn mask #48903
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 4 commits
6d3618c
2f91c6a
e7aba70
868c6d5
0f77400
f042b2e
233d055
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -852,15 +852,13 @@ def forward( | |
| position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens | ||
| position_ids = position_ids.unsqueeze(0) | ||
|
|
||
| causal_mask = None | ||
| if attention_mask is not None: | ||
| causal_mask = create_causal_mask( | ||
| config=self.config, | ||
| inputs_embeds=inputs_embeds, | ||
| attention_mask=attention_mask, | ||
| past_key_values=past_key_values, | ||
| position_ids=position_ids, | ||
| ) | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. otherwise the fix is this one but i think the issue with eager might happen with other models as well
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should be fine, no reason to do this prev pattern
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. okay then we can probably drop the eager check no ?
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The eager check was only needed for this?
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Imo, we should never do that portion should properly handle the skips and creation and is an anti pattern to the create fn |
||
| causal_mask = create_causal_mask( | ||
| config=self.config, | ||
| inputs_embeds=inputs_embeds, | ||
| attention_mask=attention_mask, | ||
| past_key_values=past_key_values, | ||
| position_ids=position_ids, | ||
| ) | ||
|
|
||
| # embed positions | ||
| hidden_states = inputs_embeds | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1737,6 +1737,109 @@ def get_llm_pos_ids_for_vision( | |
| def get_input_embeddings(self): | ||
| return self.model.get_input_embeddings() | ||
|
|
||
| def compute_3d_position_ids( | ||
| self, | ||
| inputs_tensor: torch.Tensor, | ||
| talker_input_ids: torch.LongTensor | None = None, | ||
| attention_mask: torch.Tensor | None = None, | ||
| past_key_values: Cache | None = None, | ||
| image_grid_thw: torch.Tensor | None = None, | ||
| video_grid_thw: torch.Tensor | None = None, | ||
| use_audio_in_video: bool | None = None, | ||
| audio_feature_lengths: torch.LongTensor | None = None, | ||
| video_second_per_grid: torch.Tensor | None = None, | ||
| ) -> torch.Tensor: | ||
| batch_size, seq_length = inputs_tensor.shape[:2] | ||
| past_length = 0 if past_key_values is None else past_key_values.get_seq_length() | ||
| has_multimodal_data = any(grid is not None and grid.numel() > 0 for grid in (image_grid_thw, video_grid_thw)) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| if past_length == 0 or self.rope_deltas is None: | ||
| if attention_mask is None: | ||
| attention_mask = torch.ones((batch_size, seq_length), dtype=torch.long, device=inputs_tensor.device) | ||
| if talker_input_ids is not None and has_multimodal_data: | ||
| position_ids, rope_deltas = self.get_rope_index( | ||
| talker_input_ids, | ||
| image_grid_thw, | ||
| video_grid_thw, | ||
| attention_mask, | ||
| use_audio_in_video or False, | ||
| audio_feature_lengths, | ||
| video_second_per_grid, | ||
| ) | ||
| else: | ||
| position_ids = attention_mask.long().cumsum(-1) - 1 | ||
| position_ids = position_ids.masked_fill(attention_mask == 0, 0) | ||
| position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) | ||
| rope_deltas = torch.zeros(batch_size, 1, dtype=torch.long, device=inputs_tensor.device) | ||
| self.rope_deltas = rope_deltas - (1 - attention_mask).sum(dim=-1).unsqueeze(1) | ||
| else: | ||
| delta = (past_length + self.rope_deltas).to(inputs_tensor.device) | ||
| position_ids = torch.arange(seq_length, device=inputs_tensor.device) | ||
| position_ids = position_ids.view(1, -1).expand(batch_size, -1).add(delta) | ||
| position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) | ||
| return position_ids | ||
|
|
||
| def _prepare_position_ids_for_generation(self, inputs_tensor, model_kwargs): | ||
| # Overwritten -- requires 3D position ids | ||
|
|
||
|
Comment on lines
+1783
to
+1785
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. oh i love qwen! |
||
| text_positions = super()._prepare_position_ids_for_generation(inputs_tensor, model_kwargs) | ||
| attention_mask = model_kwargs.get("attention_mask") | ||
| padding_length = (1 - attention_mask).sum(dim=-1, keepdim=True) if attention_mask is not None else 0 | ||
|
|
||
| # Early exit in case we are continuing generation from past kv | ||
| past_length = 0 | ||
| if (cache := model_kwargs.get("past_key_values")) is not None: | ||
| past_length = cache.get_seq_length() | ||
| if past_length != 0 and self.rope_deltas is not None: | ||
| # Talker stores deltas relative to the physical cache length, which includes padding. | ||
| vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) + self.rope_deltas + padding_length | ||
| position_ids = torch.cat([text_positions[None, ...], vision_positions], dim=0) | ||
| return position_ids | ||
|
|
||
| # Otherwise compute 3d position ids for audio/vision tokens and concat with text position ids | ||
| if "talker_input_ids" in model_kwargs and model_kwargs["talker_input_ids"] is not None: | ||
| inputs_tensor = model_kwargs["talker_input_ids"] | ||
|
|
||
| is_input_ids = len(inputs_tensor.shape) == 2 and inputs_tensor.dtype in [torch.int, torch.long] | ||
| has_multimodal_data = any( | ||
| grid is not None and grid.numel() > 0 | ||
| for grid in (model_kwargs.get("image_grid_thw"), model_kwargs.get("video_grid_thw")) | ||
| ) | ||
| if is_input_ids and attention_mask is not None and has_multimodal_data: | ||
| vision_positions, rope_deltas = self.get_rope_index( | ||
| inputs_tensor, | ||
| image_grid_thw=model_kwargs.get("image_grid_thw"), | ||
| video_grid_thw=model_kwargs.get("video_grid_thw"), | ||
| attention_mask=attention_mask, | ||
| use_audio_in_video=model_kwargs.get("use_audio_in_video") or False, | ||
| audio_seqlens=model_kwargs.get("audio_feature_lengths"), | ||
| second_per_grids=model_kwargs.get("video_second_per_grid"), | ||
| ) | ||
| self.rope_deltas = rope_deltas - padding_length | ||
| else: | ||
| vision_positions = text_positions.unsqueeze(0).expand(3, -1, -1) | ||
| self.rope_deltas = ( | ||
| torch.zeros(inputs_tensor.shape[0], 1, dtype=torch.long, device=inputs_tensor.device) - padding_length | ||
| ) | ||
|
|
||
| # Concatenate "text + vision" positions into [4, bs, seq-len] | ||
| text_positions = text_positions[None, ...] | ||
| position_ids = torch.cat([text_positions, vision_positions], dim=0) | ||
|
|
||
| return position_ids | ||
|
|
||
| def _expand_inputs_for_generation(self, expand_size=1, is_encoder_decoder=False, input_ids=None, **model_kwargs): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ig this comes from other qwen vls? at least the pattern
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. would like Raushan's opinion on this |
||
| position_ids = model_kwargs.pop("position_ids", None) | ||
| input_ids, model_kwargs = super()._expand_inputs_for_generation( | ||
| expand_size=expand_size, | ||
| is_encoder_decoder=is_encoder_decoder, | ||
| input_ids=input_ids, | ||
| **model_kwargs, | ||
| ) | ||
| if position_ids is not None: | ||
| batch_dim = 1 if position_ids.ndim == 3 else 0 | ||
| model_kwargs["position_ids"] = position_ids.repeat_interleave(expand_size, dim=batch_dim) | ||
| return input_ids, model_kwargs | ||
|
Comment on lines
+1832
to
+1843
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this defi doesn't work well for images/videos 😆 old hidden bug, but let it be - this model is almost never used atm, and we'll remove all overriden
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. okay XD |
||
|
|
||
| def forward( | ||
| self, | ||
| input_ids=None, | ||
|
|
@@ -1782,27 +1885,17 @@ def forward( | |
| generation_step = -1 | ||
| residual_codes = None | ||
| if position_ids is None: | ||
| past_key_values_length = 0 if past_key_values is None else past_key_values.get_seq_length() | ||
| if past_key_values_length == 0 or self.rope_deltas is None: | ||
| delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1) | ||
| position_ids, rope_deltas = self.get_rope_index( | ||
| talker_input_ids, | ||
| image_grid_thw, | ||
| video_grid_thw, | ||
| attention_mask, | ||
| use_audio_in_video, | ||
| audio_feature_lengths, | ||
| video_second_per_grid, | ||
| ) | ||
| rope_deltas = rope_deltas - delta0 | ||
| self.rope_deltas = rope_deltas | ||
| else: | ||
| batch_size, seq_length = input_ids.shape | ||
| delta = (past_key_values_length + self.rope_deltas).to(input_ids.device) | ||
| position_ids = torch.arange(seq_length, device=input_ids.device) | ||
| position_ids = position_ids.view(1, -1).expand(batch_size, -1) | ||
| position_ids = position_ids.add(delta) | ||
| position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) | ||
| position_ids = self.compute_3d_position_ids( | ||
| inputs_embeds, | ||
| talker_input_ids=talker_input_ids, | ||
| attention_mask=attention_mask, | ||
| past_key_values=past_key_values, | ||
| image_grid_thw=image_grid_thw, | ||
| video_grid_thw=video_grid_thw, | ||
| use_audio_in_video=use_audio_in_video, | ||
| audio_feature_lengths=audio_feature_lengths, | ||
| video_second_per_grid=video_second_per_grid, | ||
| ) | ||
|
|
||
| outputs: MoeModelOutputWithPast = self.model( | ||
| input_ids=None, | ||
|
|
@@ -1880,9 +1973,6 @@ def prepare_inputs_for_generation( | |
| **kwargs, | ||
| ) | ||
|
|
||
| # Qwen3-Omni will prepare position ids in forward with deltas | ||
| inputs["position_ids"] = None | ||
|
|
||
| # TODO(raushan, gante): Refactor this part to a utility function | ||
| if not is_first_iteration and kwargs.get("use_cache", True): | ||
| input_ids = input_ids[:, -1:] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should do that @Cyrilvallez ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
hmm I know flex attn was never actually used for inference but was it considered at some point? 😢