Skip to content
1 change: 0 additions & 1 deletion src/transformers/models/biogpt/modeling_biogpt.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
_CONFIG_FOR_DOC = "BioGptConfig"


# Copied from transformers.models.opt.modeling_opt.OPTLearnedPositionalEmbedding with OPT->BioGpt

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's just add a TODO here or also update that model

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should I return the comment as well? it causes a fail when using make fixup.
I tried to look to update BioGPT, but it seems to be a lot of code from different models, so i didn't know if i should have touched it. I can work on it next.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes !

Comment thread
avishaiElmakies marked this conversation as resolved.
Resolved
class BioGptLearnedPositionalEmbedding(nn.Embedding):
"""
This module learns positional embeddings up to a fixed maximum size.
Expand Down
75 changes: 64 additions & 11 deletions src/transformers/models/opt/modeling_opt.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@
_SEQ_CLASS_EXPECTED_OUTPUT = "'LABEL_0'"


def _attention_to_position_ids(attention_mask: torch.Tensor) -> torch.Tensor:
"""
Converts an attention mask to position ids.
"""

position_ids = torch.cumsum(attention_mask, dim=1)
return (position_ids * attention_mask - 1).long()
Comment thread
avishaiElmakies marked this conversation as resolved.
Resolved


class OPTLearnedPositionalEmbedding(nn.Embedding):
"""
This module learns positional embeddings up to a fixed maximum size.
Expand All @@ -71,17 +80,10 @@ def __init__(self, num_embeddings: int, embedding_dim: int):
self.offset = 2
super().__init__(num_embeddings + self.offset, embedding_dim)

def forward(self, attention_mask: torch.LongTensor, past_key_values_length: int = 0):
def forward(self, position_ids: torch.LongTensor):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that is kind of a breaking change for this module 😓

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this a problem? the weights are the same so loading should work. and this module should not be used by outside code so it is not supposed to break anything.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah but it has caused issue in the past 😉

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, how do you think I should do it? The module should get position_ids to work with packed sentences. Should I add position_ids as the last argument with None as default?

"""`input_ids_shape` is expected to be [bsz x seqlen]."""
attention_mask = attention_mask.long()

# create positions depending on attention_mask
positions = (torch.cumsum(attention_mask, dim=1).type_as(attention_mask) * attention_mask).long() - 1

# cut positions if `past_key_values_length` is > 0
positions = positions[:, past_key_values_length:]

return super().forward(positions + self.offset)
return super().forward(position_ids + self.offset)


class OPTAttention(nn.Module):
Expand Down Expand Up @@ -123,6 +125,9 @@ def forward(
self,
hidden_states: torch.Tensor,
key_value_states: Optional[torch.Tensor] = None,
position_ids: Optional[
torch.Tensor
] = None, # isn't needed in normal attention, but needed in flash attention so to keep the signature same
Comment thread
avishaiElmakies marked this conversation as resolved.
Resolved
past_key_value: Optional[Tuple[torch.Tensor]] = None,
attention_mask: Optional[torch.Tensor] = None,
layer_head_mask: Optional[torch.Tensor] = None,
Expand Down Expand Up @@ -260,6 +265,7 @@ def forward(
self,
hidden_states: torch.Tensor,
key_value_states: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
Comment thread
avishaiElmakies marked this conversation as resolved.
Resolved
past_key_value: Optional[Tuple[torch.Tensor]] = None,
attention_mask: Optional[torch.Tensor] = None,
layer_head_mask: Optional[torch.Tensor] = None,
Expand Down Expand Up @@ -345,6 +351,7 @@ def forward(
value_states,
attention_mask,
query_length,
position_ids=position_ids,
dropout=attn_dropout,
is_causal=self.is_causal,
use_top_left_mask=self._flash_attn_uses_top_left_mask,
Expand Down Expand Up @@ -387,6 +394,7 @@ def forward(
self,
hidden_states: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
layer_head_mask: Optional[torch.Tensor] = None,
past_key_value: Optional[Tuple[torch.Tensor]] = None,
output_attentions: Optional[bool] = False,
Expand Down Expand Up @@ -418,6 +426,7 @@ def forward(
hidden_states, self_attn_weights, present_key_value = self.self_attn(
hidden_states=hidden_states,
past_key_value=past_key_value,
position_ids=position_ids,
attention_mask=attention_mask,
layer_head_mask=layer_head_mask,
output_attentions=output_attentions,
Expand Down Expand Up @@ -528,6 +537,11 @@ def _init_weights(self, module):
If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
information on the default strategy.
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
config.n_positions - 1]`. for padding use -1.

[What are position IDs?](../glossary#position-ids)
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:

Expand Down Expand Up @@ -619,6 +633,7 @@ def forward(
self,
input_ids: torch.LongTensor = None,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
head_mask: Optional[torch.Tensor] = None,
past_key_values: Optional[List[torch.FloatTensor]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
Expand All @@ -644,6 +659,12 @@ def forward(
- 0 for tokens that are **masked**.

[What are attention masks?](../glossary#attention-mask)

position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
config.n_positions - 1]`. for padding use -1.

[What are position IDs?](../glossary#position-ids)
head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

Expand Down Expand Up @@ -723,7 +744,12 @@ def forward(
attention_mask, input_shape, inputs_embeds, past_key_values_length
)

pos_embeds = self.embed_positions(attention_mask, past_key_values_length)
if position_ids is None:
position_ids = _attention_to_position_ids(attention_mask)
# cut positions if `past_key_values_length` is > 0
position_ids = position_ids[:, past_key_values_length:]
Comment thread
avishaiElmakies marked this conversation as resolved.
Resolved

pos_embeds = self.embed_positions(position_ids)

if self.project_in is not None:
inputs_embeds = self.project_in(inputs_embeds)
Expand Down Expand Up @@ -768,6 +794,7 @@ def forward(
decoder_layer.__call__,
hidden_states,
causal_attention_mask,
position_ids,
head_mask[idx] if head_mask is not None else None,
None,
output_attentions,
Expand All @@ -777,6 +804,7 @@ def forward(
layer_outputs = decoder_layer(
hidden_states,
attention_mask=causal_attention_mask,
position_ids=position_ids,
layer_head_mask=(head_mask[idx] if head_mask is not None else None),
past_key_value=past_key_value,
output_attentions=output_attentions,
Expand Down Expand Up @@ -843,6 +871,7 @@ def forward(
self,
input_ids: torch.LongTensor = None,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
head_mask: Optional[torch.Tensor] = None,
past_key_values: Optional[List[torch.FloatTensor]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
Expand All @@ -862,6 +891,7 @@ def forward(
decoder_outputs = self.decoder(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
past_key_values=past_key_values,
inputs_embeds=inputs_embeds,
Expand Down Expand Up @@ -918,6 +948,7 @@ def forward(
self,
input_ids: torch.LongTensor = None,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
head_mask: Optional[torch.Tensor] = None,
past_key_values: Optional[List[torch.FloatTensor]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
Expand All @@ -944,6 +975,11 @@ def forward(
- 0 for tokens that are **masked**.

[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
config.n_positions - 1]`. for padding use -1.

[What are position IDs?](../glossary#position-ids)
head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

Expand Down Expand Up @@ -1011,6 +1047,7 @@ def forward(
outputs = self.model.decoder(
input_ids=input_ids,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
past_key_values=past_key_values,
inputs_embeds=inputs_embeds,
Expand Down Expand Up @@ -1046,7 +1083,7 @@ def forward(
)

def prepare_inputs_for_generation(
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, position_ids=None, **kwargs
):
if past_key_values is not None:
past_length = past_key_values[0][0].shape[2]
Expand All @@ -1060,6 +1097,17 @@ def prepare_inputs_for_generation(

input_ids = input_ids[:, remove_prefix_length:]

if attention_mask is not None and position_ids is None:
position_ids = _attention_to_position_ids(attention_mask)
if past_key_values:
position_ids = position_ids[:, -input_ids.shape[1] :]

# This `clone` call is needed to avoid recapturing cuda graphs with `torch.compile`'s `mode="reduce-overhead`,
# as otherwise the input `position_ids` would have various stride during the decoding.
# Here, simply using `.contiguous()` is not sufficient as in the batch size = 1 case,
# position_ids` is already contiguous but with varying stride which retriggers a capture.
position_ids = position_ids.clone(memory_format=torch.contiguous_format)

# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
if inputs_embeds is not None and past_key_values is None:
model_inputs = {"inputs_embeds": inputs_embeds}
Expand All @@ -1071,6 +1119,7 @@ def prepare_inputs_for_generation(
"past_key_values": past_key_values,
"use_cache": kwargs.get("use_cache"),
"attention_mask": attention_mask,
"position_ids": position_ids,
}
)
return model_inputs
Expand Down Expand Up @@ -1122,6 +1171,7 @@ def forward(
self,
input_ids: Optional[torch.LongTensor] = None,
attention_mask: Optional[torch.FloatTensor] = None,
position_ids: Optional[torch.LongTensor] = None,
head_mask: Optional[torch.FloatTensor] = None,
past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
Expand All @@ -1143,6 +1193,7 @@ def forward(
input_ids,
past_key_values=past_key_values,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
use_cache=use_cache,
Expand Down Expand Up @@ -1238,6 +1289,7 @@ def forward(
self,
input_ids: Optional[torch.LongTensor] = None,
attention_mask: Optional[torch.FloatTensor] = None,
position_ids: Optional[torch.LongTensor] = None,
head_mask: Optional[torch.FloatTensor] = None,
past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
Expand Down Expand Up @@ -1297,6 +1349,7 @@ def forward(
input_ids,
past_key_values=past_key_values,
attention_mask=attention_mask,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds,
use_cache=use_cache,
Expand Down