Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/transformers/models/imagegpt/modeling_imagegpt.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ def forward(
"If class is used as cross attention, the weights `q_attn` have to be defined. "
"Please make sure to instantiate class with `ImageGPTAttention(..., is_cross_attention=True)`."
)
attention_mask = encoder_attention_mask

if layer_past is not None and is_updated:
# reuse k,v, cross_attentions, and compute only q
Expand All @@ -244,7 +245,9 @@ def forward(
key = key.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2)
value = value.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2)

if layer_past is not None:
if (layer_past is not None and not is_cross_attention) or (
layer_past is not None and is_cross_attention and not is_updated
):
Comment thread
blipbyte marked this conversation as resolved.
Resolved
# save all key/value_states to cache to be re-used for fast auto-regressive generation
key, value = curr_past_key_values.update(key, value, self.layer_idx)
# set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls
Expand Down Expand Up @@ -493,8 +496,12 @@ def forward(
if token_type_ids is not None:
token_type_ids = token_type_ids.view(-1, input_shape[-1])

if use_cache and past_key_values is None:
past_key_values = DynamicCache(config=self.config)
if use_cache:
if past_key_values is None:
past_key_values = DynamicCache(config=self.config)

if self.config.add_cross_attention and not isinstance(past_key_values, EncoderDecoderCache):
past_key_values = EncoderDecoderCache(past_key_values, DynamicCache(config=self.config))

if position_ids is None:
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
Expand Down
120 changes: 119 additions & 1 deletion tests/models/imagegpt/test_modeling_imagegpt.py
Comment thread
blipbyte marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


Expand Down Expand Up @@ -292,6 +292,124 @@ def test_forward_signature(self):
expected_arg_names = ["input_ids"]
self.assertListEqual(arg_names[:1], expected_arg_names)

def test_cross_attention_respects_encoder_padding_mask(self):
torch.manual_seed(0)
config, input_ids = self.model_tester.prepare_config_and_inputs()[:2]
config.add_cross_attention = True
model = ImageGPTModel(config).to(torch_device).eval()

encoder_seq_length = self.model_tester.seq_length + 2
batch_size = self.model_tester.batch_size
encoder_hidden_states = floats_tensor([batch_size, encoder_seq_length, self.model_tester.hidden_size]).to(
torch_device
)
encoder_attention_mask = torch.ones(batch_size, encoder_seq_length, dtype=torch.long, device=torch_device)
padding = encoder_seq_length // 2
encoder_attention_mask[:, padding:] = 0

# What the mask hides must not move the output; what it leaves visible must, or the check is vacuous
corrupted_encoder_states = encoder_hidden_states.clone()
corrupted_encoder_states[:, padding:] = 1e6
visible_corrupted_encoder_states = encoder_hidden_states.clone()
visible_corrupted_encoder_states[:, :padding] = 1e6

with torch.no_grad():
output = model(
input_ids=input_ids,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
).last_hidden_state
output_corrupted = model(
input_ids=input_ids,
encoder_hidden_states=corrupted_encoder_states,
encoder_attention_mask=encoder_attention_mask,
).last_hidden_state
output_visible_corrupted = model(
input_ids=input_ids,
encoder_hidden_states=visible_corrupted_encoder_states,
encoder_attention_mask=encoder_attention_mask,
).last_hidden_state

self.assertTrue(torch.equal(output, output_corrupted))
self.assertFalse(torch.equal(output, output_visible_corrupted))

def test_cross_attention_with_padded_decoder_attention_mask(self):
torch.manual_seed(0)
config, input_ids = self.model_tester.prepare_config_and_inputs()[:2]
config.add_cross_attention = True
model = ImageGPTModel(config).to(torch_device).eval()

batch_size, seq_length = input_ids.shape
encoder_seq_length = seq_length + 2
encoder_hidden_states = floats_tensor([batch_size, encoder_seq_length, self.model_tester.hidden_size]).to(
torch_device
)
encoder_attention_mask = torch.ones(batch_size, encoder_seq_length, dtype=torch.long, device=torch_device)
padding = encoder_seq_length // 2
encoder_attention_mask[:, padding:] = 0

attention_mask = torch.ones(batch_size, seq_length, dtype=torch.long, device=torch_device)
attention_mask[:, -2:] = 0

corrupted_encoder_states = encoder_hidden_states.clone()
corrupted_encoder_states[:, padding:] = 1e6

with torch.no_grad():
output = model(
input_ids=input_ids,
attention_mask=attention_mask,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
).last_hidden_state
output_corrupted = model(
input_ids=input_ids,
attention_mask=attention_mask,
encoder_hidden_states=corrupted_encoder_states,
encoder_attention_mask=encoder_attention_mask,
).last_hidden_state

self.assertEqual(output.shape, (batch_size, seq_length, self.model_tester.hidden_size))
self.assertTrue(torch.equal(output, output_corrupted))

def test_cross_attention_cached_decoding_matches_uncached(self):
torch.manual_seed(0)
config, input_ids = self.model_tester.prepare_config_and_inputs()[:2]
config.add_cross_attention = True
model = ImageGPTModel(config).to(torch_device).eval()

batch_size, seq_length = input_ids.shape
encoder_seq_length = seq_length + 2
encoder_hidden_states = floats_tensor([batch_size, encoder_seq_length, self.model_tester.hidden_size]).to(
torch_device
)
encoder_attention_mask = torch.ones(batch_size, encoder_seq_length, dtype=torch.long, device=torch_device)
encoder_attention_mask[:, encoder_seq_length // 2 :] = 0

with torch.no_grad():
reference = model(
input_ids=input_ids,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
use_cache=False,
).last_hidden_state
prefill = model(
input_ids=input_ids[:, :-1],
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
use_cache=True,
)
step = model(
input_ids=input_ids[:, -1:],
past_key_values=prefill.past_key_values,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
use_cache=True,
)

cross_attention_keys = step.past_key_values.cross_attention_cache.layers[0].keys
self.assertEqual(cross_attention_keys.shape[-2], encoder_seq_length)
torch.testing.assert_close(step.last_hidden_state[:, -1], reference[:, -1], rtol=1e-4, atol=1e-4)

@unittest.skip(reason="Model inputs don't fit test pattern") # and it's not used enough to be worth fixing :)
def test_past_key_values_format(self):
pass
Expand Down