diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index 3e642688a1ec..f7dd492a97d4 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -228,6 +228,7 @@ def forward( "If class is used as cross attention, the weights `q_attn` have to be defined. " "Please make sure to instantiate class with `ImageGPTAttention(..., is_cross_attention=True)`." ) + attention_mask = encoder_attention_mask if layer_past is not None and is_updated: # reuse k,v, cross_attentions, and compute only q @@ -244,7 +245,7 @@ def forward( key = key.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) value = value.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - if layer_past is not None: + if layer_past is not None and not (is_cross_attention and is_updated): # save all key/value_states to cache to be re-used for fast auto-regressive generation key, value = curr_past_key_values.update(key, value, self.layer_idx) # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls @@ -493,8 +494,12 @@ def forward( if token_type_ids is not None: token_type_ids = token_type_ids.view(-1, input_shape[-1]) - if use_cache and past_key_values is None: - past_key_values = DynamicCache(config=self.config) + if use_cache: + if past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if self.config.add_cross_attention and not isinstance(past_key_values, EncoderDecoderCache): + past_key_values = EncoderDecoderCache(past_key_values, DynamicCache(config=self.config)) if position_ids is None: past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 diff --git a/tests/models/imagegpt/test_modeling_imagegpt.py b/tests/models/imagegpt/test_modeling_imagegpt.py index 5b40258fc2a4..672cbada575d 100644 --- a/tests/models/imagegpt/test_modeling_imagegpt.py +++ b/tests/models/imagegpt/test_modeling_imagegpt.py @@ -23,7 +23,7 @@ from ...generation.test_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester -from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask +from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask from ...test_pipeline_mixin import PipelineTesterMixin @@ -292,6 +292,56 @@ def test_forward_signature(self): expected_arg_names = ["input_ids"] self.assertListEqual(arg_names[:1], expected_arg_names) + def test_cross_attention(self): + config, input_ids, attention_mask = self.model_tester.prepare_config_and_inputs()[:3] + config.add_cross_attention = True + model = ImageGPTModel(config).to(torch_device).eval() + + batch_size, seq_length = input_ids.shape + encoder_seq_length = seq_length + 2 + padding = encoder_seq_length // 2 + encoder_hidden_states = floats_tensor([batch_size, encoder_seq_length, self.model_tester.hidden_size]).to( + torch_device + ) + encoder_attention_mask = torch.ones(batch_size, encoder_seq_length, dtype=torch.long, device=torch_device) + encoder_attention_mask[:, padding:] = 0 + + # Masked encoder positions must not affect the output; visible ones must, or the check is vacuous + masked_corrupted = encoder_hidden_states.clone() + masked_corrupted[:, padding:] = 1e6 + visible_corrupted = encoder_hidden_states.clone() + visible_corrupted[:, :padding] = 1e6 + + with torch.no_grad(): + outputs = [ + model( + input_ids, + attention_mask=attention_mask, + encoder_hidden_states=states, + encoder_attention_mask=encoder_attention_mask, + ).last_hidden_state + for states in (encoder_hidden_states, masked_corrupted, visible_corrupted) + ] + prefill = model( + input_ids[:, :-1], + attention_mask=attention_mask[:, :-1], + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + ) + step = model( + input_ids[:, -1:], + past_key_values=prefill.past_key_values, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + ) + + output, output_masked_corrupted, output_visible_corrupted = outputs + self.assertTrue(torch.equal(output, output_masked_corrupted)) + self.assertFalse(torch.equal(output, output_visible_corrupted)) + self.assertEqual(step.past_key_values.cross_attention_cache.get_seq_length(), encoder_seq_length) + torch.testing.assert_close(step.last_hidden_state[:, -1], output[:, -1], rtol=1e-4, atol=1e-4) + @unittest.skip(reason="Model inputs don't fit test pattern") # and it's not used enough to be worth fixing :) def test_past_key_values_format(self): pass