Repository navigation
Add position ids in forward pass to opt model #33121
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 4 commits
ecddd51
0d52209
b0ff1b5
a19baed
3ae719a
c5facaf
48299c6
e27a913
a9a9d9e
9ca9a9f
42e27f6
466df4b
376b184
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -60,6 +60,15 @@ | |
| _SEQ_CLASS_EXPECTED_OUTPUT = "'LABEL_0'" | ||
|
|
||
|
|
||
| def _attention_to_position_ids(attention_mask: torch.Tensor) -> torch.Tensor: | ||
| """ | ||
| Converts an attention mask to position ids. | ||
| """ | ||
|
|
||
| position_ids = torch.cumsum(attention_mask, dim=1) | ||
| return (position_ids * attention_mask - 1).long() | ||
|
avishaiElmakies marked this conversation as resolved.
Resolved
|
||
|
|
||
|
|
||
| class OPTLearnedPositionalEmbedding(nn.Embedding): | ||
| """ | ||
| This module learns positional embeddings up to a fixed maximum size. | ||
|
|
@@ -71,17 +80,10 @@ def __init__(self, num_embeddings: int, embedding_dim: int): | |
| self.offset = 2 | ||
| super().__init__(num_embeddings + self.offset, embedding_dim) | ||
|
|
||
| def forward(self, attention_mask: torch.LongTensor, past_key_values_length: int = 0): | ||
| def forward(self, position_ids: torch.LongTensor): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. that is kind of a breaking change for this module 😓
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why is this a problem? the weights are the same so loading should work. and this module should not be used by outside code so it is not supposed to break anything.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah but it has caused issue in the past 😉
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, how do you think I should do it? The module should get position_ids to work with packed sentences. Should I add position_ids as the last argument with None as default? |
||
| """`input_ids_shape` is expected to be [bsz x seqlen].""" | ||
| attention_mask = attention_mask.long() | ||
|
|
||
| # create positions depending on attention_mask | ||
| positions = (torch.cumsum(attention_mask, dim=1).type_as(attention_mask) * attention_mask).long() - 1 | ||
|
|
||
| # cut positions if `past_key_values_length` is > 0 | ||
| positions = positions[:, past_key_values_length:] | ||
|
|
||
| return super().forward(positions + self.offset) | ||
| return super().forward(position_ids + self.offset) | ||
|
|
||
|
|
||
| class OPTAttention(nn.Module): | ||
|
|
@@ -123,6 +125,9 @@ def forward( | |
| self, | ||
| hidden_states: torch.Tensor, | ||
| key_value_states: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[ | ||
| torch.Tensor | ||
| ] = None, # isn't needed in normal attention, but needed in flash attention so to keep the signature same | ||
|
avishaiElmakies marked this conversation as resolved.
Resolved
|
||
| past_key_value: Optional[Tuple[torch.Tensor]] = None, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| layer_head_mask: Optional[torch.Tensor] = None, | ||
|
|
@@ -260,6 +265,7 @@ def forward( | |
| self, | ||
| hidden_states: torch.Tensor, | ||
| key_value_states: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[torch.Tensor] = None, | ||
|
avishaiElmakies marked this conversation as resolved.
Resolved
|
||
| past_key_value: Optional[Tuple[torch.Tensor]] = None, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| layer_head_mask: Optional[torch.Tensor] = None, | ||
|
|
@@ -345,6 +351,7 @@ def forward( | |
| value_states, | ||
| attention_mask, | ||
| query_length, | ||
| position_ids=position_ids, | ||
| dropout=attn_dropout, | ||
| is_causal=self.is_causal, | ||
| use_top_left_mask=self._flash_attn_uses_top_left_mask, | ||
|
|
@@ -387,6 +394,7 @@ def forward( | |
| self, | ||
| hidden_states: torch.Tensor, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| layer_head_mask: Optional[torch.Tensor] = None, | ||
| past_key_value: Optional[Tuple[torch.Tensor]] = None, | ||
| output_attentions: Optional[bool] = False, | ||
|
|
@@ -418,6 +426,7 @@ def forward( | |
| hidden_states, self_attn_weights, present_key_value = self.self_attn( | ||
| hidden_states=hidden_states, | ||
| past_key_value=past_key_value, | ||
| position_ids=position_ids, | ||
| attention_mask=attention_mask, | ||
| layer_head_mask=layer_head_mask, | ||
| output_attentions=output_attentions, | ||
|
|
@@ -528,6 +537,11 @@ def _init_weights(self, module): | |
| If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] | ||
| and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more | ||
| information on the default strategy. | ||
| position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): | ||
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, | ||
| config.n_positions - 1]`. for padding use -1. | ||
|
|
||
| [What are position IDs?](../glossary#position-ids) | ||
| head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): | ||
| Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: | ||
|
|
||
|
|
@@ -619,6 +633,7 @@ def forward( | |
| self, | ||
| input_ids: torch.LongTensor = None, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| head_mask: Optional[torch.Tensor] = None, | ||
| past_key_values: Optional[List[torch.FloatTensor]] = None, | ||
| inputs_embeds: Optional[torch.FloatTensor] = None, | ||
|
|
@@ -644,6 +659,12 @@ def forward( | |
| - 0 for tokens that are **masked**. | ||
|
|
||
| [What are attention masks?](../glossary#attention-mask) | ||
|
|
||
| position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): | ||
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, | ||
| config.n_positions - 1]`. for padding use -1. | ||
|
|
||
| [What are position IDs?](../glossary#position-ids) | ||
| head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): | ||
| Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: | ||
|
|
||
|
|
@@ -723,7 +744,12 @@ def forward( | |
| attention_mask, input_shape, inputs_embeds, past_key_values_length | ||
| ) | ||
|
|
||
| pos_embeds = self.embed_positions(attention_mask, past_key_values_length) | ||
| if position_ids is None: | ||
| position_ids = _attention_to_position_ids(attention_mask) | ||
| # cut positions if `past_key_values_length` is > 0 | ||
| position_ids = position_ids[:, past_key_values_length:] | ||
|
avishaiElmakies marked this conversation as resolved.
Resolved
|
||
|
|
||
| pos_embeds = self.embed_positions(position_ids) | ||
|
|
||
| if self.project_in is not None: | ||
| inputs_embeds = self.project_in(inputs_embeds) | ||
|
|
@@ -768,6 +794,7 @@ def forward( | |
| decoder_layer.__call__, | ||
| hidden_states, | ||
| causal_attention_mask, | ||
| position_ids, | ||
| head_mask[idx] if head_mask is not None else None, | ||
| None, | ||
| output_attentions, | ||
|
|
@@ -777,6 +804,7 @@ def forward( | |
| layer_outputs = decoder_layer( | ||
| hidden_states, | ||
| attention_mask=causal_attention_mask, | ||
| position_ids=position_ids, | ||
| layer_head_mask=(head_mask[idx] if head_mask is not None else None), | ||
| past_key_value=past_key_value, | ||
| output_attentions=output_attentions, | ||
|
|
@@ -843,6 +871,7 @@ def forward( | |
| self, | ||
| input_ids: torch.LongTensor = None, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| head_mask: Optional[torch.Tensor] = None, | ||
| past_key_values: Optional[List[torch.FloatTensor]] = None, | ||
| inputs_embeds: Optional[torch.FloatTensor] = None, | ||
|
|
@@ -862,6 +891,7 @@ def forward( | |
| decoder_outputs = self.decoder( | ||
| input_ids=input_ids, | ||
| attention_mask=attention_mask, | ||
| position_ids=position_ids, | ||
| head_mask=head_mask, | ||
| past_key_values=past_key_values, | ||
| inputs_embeds=inputs_embeds, | ||
|
|
@@ -918,6 +948,7 @@ def forward( | |
| self, | ||
| input_ids: torch.LongTensor = None, | ||
| attention_mask: Optional[torch.Tensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| head_mask: Optional[torch.Tensor] = None, | ||
| past_key_values: Optional[List[torch.FloatTensor]] = None, | ||
| inputs_embeds: Optional[torch.FloatTensor] = None, | ||
|
|
@@ -944,6 +975,11 @@ def forward( | |
| - 0 for tokens that are **masked**. | ||
|
|
||
| [What are attention masks?](../glossary#attention-mask) | ||
| position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): | ||
| Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, | ||
| config.n_positions - 1]`. for padding use -1. | ||
|
|
||
| [What are position IDs?](../glossary#position-ids) | ||
| head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): | ||
| Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: | ||
|
|
||
|
|
@@ -1011,6 +1047,7 @@ def forward( | |
| outputs = self.model.decoder( | ||
| input_ids=input_ids, | ||
| attention_mask=attention_mask, | ||
| position_ids=position_ids, | ||
| head_mask=head_mask, | ||
| past_key_values=past_key_values, | ||
| inputs_embeds=inputs_embeds, | ||
|
|
@@ -1046,7 +1083,7 @@ def forward( | |
| ) | ||
|
|
||
| def prepare_inputs_for_generation( | ||
| self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs | ||
| self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, position_ids=None, **kwargs | ||
| ): | ||
| if past_key_values is not None: | ||
| past_length = past_key_values[0][0].shape[2] | ||
|
|
@@ -1060,6 +1097,17 @@ def prepare_inputs_for_generation( | |
|
|
||
| input_ids = input_ids[:, remove_prefix_length:] | ||
|
|
||
| if attention_mask is not None and position_ids is None: | ||
| position_ids = _attention_to_position_ids(attention_mask) | ||
| if past_key_values: | ||
| position_ids = position_ids[:, -input_ids.shape[1] :] | ||
|
|
||
| # This `clone` call is needed to avoid recapturing cuda graphs with `torch.compile`'s `mode="reduce-overhead`, | ||
| # as otherwise the input `position_ids` would have various stride during the decoding. | ||
| # Here, simply using `.contiguous()` is not sufficient as in the batch size = 1 case, | ||
| # position_ids` is already contiguous but with varying stride which retriggers a capture. | ||
| position_ids = position_ids.clone(memory_format=torch.contiguous_format) | ||
|
|
||
| # if `inputs_embeds` are passed, we only want to use them in the 1st generation step | ||
| if inputs_embeds is not None and past_key_values is None: | ||
| model_inputs = {"inputs_embeds": inputs_embeds} | ||
|
|
@@ -1071,6 +1119,7 @@ def prepare_inputs_for_generation( | |
| "past_key_values": past_key_values, | ||
| "use_cache": kwargs.get("use_cache"), | ||
| "attention_mask": attention_mask, | ||
| "position_ids": position_ids, | ||
| } | ||
| ) | ||
| return model_inputs | ||
|
|
@@ -1122,6 +1171,7 @@ def forward( | |
| self, | ||
| input_ids: Optional[torch.LongTensor] = None, | ||
| attention_mask: Optional[torch.FloatTensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| head_mask: Optional[torch.FloatTensor] = None, | ||
| past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, | ||
| inputs_embeds: Optional[torch.FloatTensor] = None, | ||
|
|
@@ -1143,6 +1193,7 @@ def forward( | |
| input_ids, | ||
| past_key_values=past_key_values, | ||
| attention_mask=attention_mask, | ||
| position_ids=position_ids, | ||
| head_mask=head_mask, | ||
| inputs_embeds=inputs_embeds, | ||
| use_cache=use_cache, | ||
|
|
@@ -1238,6 +1289,7 @@ def forward( | |
| self, | ||
| input_ids: Optional[torch.LongTensor] = None, | ||
| attention_mask: Optional[torch.FloatTensor] = None, | ||
| position_ids: Optional[torch.LongTensor] = None, | ||
| head_mask: Optional[torch.FloatTensor] = None, | ||
| past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, | ||
| inputs_embeds: Optional[torch.FloatTensor] = None, | ||
|
|
@@ -1297,6 +1349,7 @@ def forward( | |
| input_ids, | ||
| past_key_values=past_key_values, | ||
| attention_mask=attention_mask, | ||
| position_ids=position_ids, | ||
| head_mask=head_mask, | ||
| inputs_embeds=inputs_embeds, | ||
| use_cache=use_cache, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
let's just add a TODO here or also update that model
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should I return the comment as well? it causes a fail when using make fixup.
I tried to look to update BioGPT, but it seems to be a lot of code from different models, so i didn't know if i should have touched it. I can work on it next.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yes !