Skip to content
Merged
10 changes: 7 additions & 3 deletions src/transformers/generation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,10 +378,14 @@ def prepare_inputs_for_generation(
# If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens
# Exception 1: when passing input_embeds, input_ids may be missing entries
# Exception 2: some generation methods do special slicing of input_ids, so we don't need to do it here
# Exception 3: with synced GPUs cache_position may go out of bounds, but we only want dummy token in that case
# Exception 3: with synced GPUs cache_position may go out of bounds, but we only want dummy token in that case.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(changes in this file were done for end-to-end compilation)

# (we can't check exception 3 while compiling)
if past_key_values is not None:
model_inputs["past_key_values"] = past_key_values
if inputs_embeds is not None or cache_position[-1] >= input_ids.shape[1]: # Exception 1 or Exception 3
if (
inputs_embeds is not None # Exception 1
or (is_torchdynamo_compiling() or cache_position[-1] >= input_ids.shape[1]) # Exception 3
):
input_ids = input_ids[:, -cache_position.shape[0] :]
elif input_ids.shape[1] != cache_position.shape[0]: # Default case (the "else", a no op, is Exception 2)
input_ids = input_ids[:, cache_position]
Expand Down Expand Up @@ -414,7 +418,7 @@ def prepare_inputs_for_generation(
for model_input_name in ["position_ids", "token_type_ids"]:
model_input = kwargs.get(model_input_name)
if model_input is not None:
if past_key_values:
if past_key_values is not None:
model_input = model_input[:, -input_ids.shape[1] :]
model_input = model_input.clone(memory_format=torch.contiguous_format)
model_inputs[model_input_name] = model_input
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -911,7 +911,8 @@ def forward(

if (pixel_values is not None or pixel_values_videos is not None) and inputs_embeds is not None:
raise ValueError(
"You cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one"
"You cannot specify both `pixel_values`/`pixel_values_videos` and `inputs_embeds` at the same time, "
"and must specify either one"
)

legacy_processing = False
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,8 @@ def forward(

if (pixel_values is not None or pixel_values_videos is not None) and inputs_embeds is not None:
raise ValueError(
"You cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one"
"You cannot specify both `pixel_values`/`pixel_values_videos` and `inputs_embeds` at the same time, "
"and must specify either one"
)

legacy_processing = False
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,8 @@ def forward(

if (pixel_values is not None or pixel_values_videos is not None) and inputs_embeds is not None:
raise ValueError(
"You cannot specify both pixel_values/pixel_values_videos and inputs_embeds at the same time, and must specify either one"
"You cannot specify both `pixel_values`/`pixel_values_videos` and `inputs_embeds` at the same time, "
"and must specify either one"
)

if inputs_embeds is None:
Expand Down
3 changes: 2 additions & 1 deletion src/transformers/models/video_llava/modeling_video_llava.py
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,8 @@ def forward(

if (pixel_values_images is not None or pixel_values_videos is not None) and inputs_embeds is not None:
raise ValueError(
"You cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one"
"You cannot specify both `pixel_values_images`/`pixel_values_videos` and `inputs_embeds` at the same "
"time, and must specify either one"
)

legacy_processing = False
Expand Down
346 changes: 188 additions & 158 deletions tests/generation/test_utils.py

Large diffs are not rendered by default.

5 changes: 0 additions & 5 deletions tests/models/bart/test_modeling_bart.py
Original file line number Diff line number Diff line change
Expand Up @@ -1532,8 +1532,3 @@ def test_retain_grad_hidden_states_attentions(self):
@unittest.skip
def test_save_load_fast_init_from_base(self):
pass

@unittest.skip(reason="Generate needs input ids")
def test_inputs_embeds_matches_input_ids_with_generate(self):
# generate only works with input ids for bartforcausalLM
pass
5 changes: 0 additions & 5 deletions tests/models/bert/test_modeling_bert.py
Original file line number Diff line number Diff line change
Expand Up @@ -509,11 +509,6 @@ def test_model_as_decoder(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
self.model_tester.create_and_check_model_as_decoder(*config_and_inputs)

@unittest.skip(reason="Generate needs input ids")
def test_inputs_embeds_matches_input_ids_with_generate(self):
# generate only works with input ids for bertforcausalLM
pass

def test_model_as_decoder_with_default_input_mask(self):
# This regression test was failing with PyTorch < 1.3
(
Expand Down
40 changes: 0 additions & 40 deletions tests/models/chameleon/test_modeling_chameleon.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,14 @@

import unittest

import pytest
import requests
from parameterized import parameterized

from transformers import ChameleonConfig, is_torch_available, is_vision_available, set_seed
from transformers.testing_utils import (
require_bitsandbytes,
require_flash_attn,
require_read_token,
require_torch,
require_torch_gpu,
slow,
torch_device,
)
Expand Down Expand Up @@ -329,43 +326,6 @@ def test_model_rope_scaling(self, scaling_type):
# The output should be different for long inputs
self.assertFalse(torch.allclose(original_long_output, scaled_long_output, atol=1e-5))

@require_flash_attn
@require_read_token
@require_torch_gpu
@require_bitsandbytes
@pytest.mark.flash_attn_test
@slow
def test_flash_attn_2_generate_padding_right(self):
"""
Overwritting the common test as the test is flaky on tiny models
"""
model = ChameleonForConditionalGeneration.from_pretrained(
"facebook/chameleon-7b",
load_in_4bit=True,
device_map={"": 0},
)

processor = ChameleonProcessor.from_pretrained("facebook/chameleon-7b")
texts = ["hi", "Hello this is a very long sentence"]

processor.tokenizer.padding_side = "right"

inputs = processor(text=texts, return_tensors="pt", padding=True).to(0)

output_native = model.generate(**inputs, max_new_tokens=20, do_sample=False)
output_native = processor.tokenizer.batch_decode(output_native)

model = ChameleonForConditionalGeneration.from_pretrained(
"facebook/chameleon-7b",
load_in_4bit=True,
attn_implementation="flash_attention_2",
)

output_fa_2 = model.generate(**inputs, max_new_tokens=20, do_sample=False)
output_fa_2 = processor.tokenizer.batch_decode(output_fa_2)

self.assertListEqual(output_native, output_fa_2)

@unittest.skip("Chameleon forces some token ids to be -inf!")
def test_batching_equivalence(self):
pass
Expand Down
48 changes: 0 additions & 48 deletions tests/models/gemma/test_modeling_gemma.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,9 +319,6 @@ class GemmaModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixi
# This is because we are hitting edge cases with the causal_mask buffer
model_split_percents = [0.5, 0.6]

# used in `test_torch_compile`
_torch_compile_test_ckpt = "google/gemma-2b"

# used in `test_torch_compile_for_training`
_torch_compile_train_cls = GemmaForCausalLM if is_torch_available() else None

Expand Down Expand Up @@ -419,51 +416,6 @@ def test_save_load_fast_init_from_base(self):
def test_past_key_values_format(self):
pass

@require_flash_attn
@require_torch_gpu
@pytest.mark.flash_attn_test
@slow
def test_flash_attn_2_generate_use_cache(self):
import torch

max_new_tokens = 30

for model_class in self.all_generative_model_classes:
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

dummy_input = inputs_dict[model_class.main_input_name]
if dummy_input.dtype in [torch.float32, torch.bfloat16]:
dummy_input = dummy_input.to(torch.float16)

# make sure that all models have enough positions for generation
if hasattr(config, "max_position_embeddings"):
config.max_position_embeddings = max_new_tokens + dummy_input.shape[1] + 1

model = model_class(config)

with tempfile.TemporaryDirectory() as tmpdirname:
model.save_pretrained(tmpdirname)

dummy_attention_mask = inputs_dict.get("attention_mask", torch.ones_like(dummy_input))
# NOTE: Gemma apparently does not support right padding + use_cache with FA2.
dummy_attention_mask[:, -1] = 1

model = model_class.from_pretrained(
tmpdirname,
torch_dtype=torch.float16,
attn_implementation="flash_attention_2",
low_cpu_mem_usage=True,
).to(torch_device)

# Just test that a large cache works as expected
_ = model.generate(
dummy_input,
attention_mask=dummy_attention_mask,
max_new_tokens=max_new_tokens,
do_sample=False,
use_cache=True,
)

@require_flash_attn
@require_torch_gpu
@pytest.mark.flash_attn_test
Expand Down
1 change: 0 additions & 1 deletion tests/models/gemma2/test_modeling_gemma2.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ class Gemma2ModelTest(GemmaModelTest, unittest.TestCase):
test_pruning = False
_is_stateful = True
model_split_percents = [0.5, 0.6]
_torch_compile_test_ckpt = "google/gemma-2-9b"

def setUp(self):
self.model_tester = Gemma2ModelTester(self)
Expand Down
40 changes: 0 additions & 40 deletions tests/models/glm/test_modeling_glm.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
require_flash_attn,
require_torch,
require_torch_accelerator,
require_torch_gpu,
require_torch_sdpa,
slow,
torch_device,
Expand Down Expand Up @@ -306,10 +305,6 @@ class GlmModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin,
test_headmasking = False
test_pruning = False

# used in `test_torch_compile`
_torch_compile_test_ckpt = "THUDM/glm-4-9b"
_torch_compile_test_revision = "refs/pr/15"

def setUp(self):
self.model_tester = GlmModelTester(self)
self.config_tester = ConfigTester(self, config_class=GlmConfig, hidden_size=37)
Expand Down Expand Up @@ -426,41 +421,6 @@ def test_custom_4d_attention_mask(self):

torch.testing.assert_close(normalized_0, normalized_1, rtol=1e-3, atol=1e-3)

@require_flash_attn
@require_torch_gpu
@pytest.mark.flash_attn_test
@slow
def test_flash_attn_2_generate_padding_right(self):
"""Overwrite the common test as the test is flaky on tiny models."""
model = GlmForCausalLM.from_pretrained(
"THUDM/glm-4-9b",
device_map={"": 0},
torch_dtype=torch.bfloat16,
revision="refs/pr/15",
)

tokenizer = AutoTokenizer.from_pretrained("THUDM/glm-4-9b", revision="refs/pr/15")
tokenizer.padding_side = "right"

texts = ["hi", "Hello this is a very long sentence"]
inputs = tokenizer(texts, return_tensors="pt", padding=True).to(0)

output_native = model.generate(**inputs, max_new_tokens=15, do_sample=False)
output_native = tokenizer.batch_decode(output_native)

model = GlmForCausalLM.from_pretrained(
"THUDM/glm-4-9b",
device_map={"": 0},
attn_implementation="flash_attention_2",
torch_dtype=torch.bfloat16,
revision="refs/pr/15",
)

output_fa_2 = model.generate(**inputs, max_new_tokens=15, do_sample=False)
output_fa_2 = tokenizer.batch_decode(output_fa_2)

self.assertListEqual(output_native, output_fa_2)

@parameterized.expand([("float16",), ("bfloat16",), ("float32",)])
@require_torch_sdpa
@slow
Expand Down
45 changes: 1 addition & 44 deletions tests/models/gptj/test_modeling_gptj.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,9 @@
import datetime
import unittest

import pytest

from transformers import BitsAndBytesConfig, GPTJConfig, is_torch_available
from transformers import GPTJConfig, is_torch_available
from transformers.testing_utils import (
require_bitsandbytes,
require_flash_attn,
require_torch,
require_torch_gpu,
slow,
tooslow,
torch_device,
Expand Down Expand Up @@ -505,44 +500,6 @@ def test_model_from_pretrained(self):
model = GPTJModel.from_pretrained(model_name, revision="float16", torch_dtype=torch.float16)
self.assertIsNotNone(model)

@require_flash_attn
@require_torch_gpu
@require_bitsandbytes
@pytest.mark.flash_attn_test
@slow
def test_flash_attn_2_generate_padding_right(self):
"""
Overwritting the common test as the test is flaky on tiny models
"""
tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")

texts = ["hi", "Hello this is a very long sentence"]
expected_outputs = [
"hi<|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|><|endoftext|>Q: I have a question about the new version of the game. I have a question about the",
"Hello this is a very long sentence.\n\nA:\n\nI think the best way to understand this is to think of it",
]

tokenizer.padding_side = "right"
tokenizer.pad_token = tokenizer.eos_token

inputs = tokenizer(texts, return_tensors="pt", padding=True).to(0)

quantization_config = BitsAndBytesConfig(load_in_4bit=True)

model = GPTJForCausalLM.from_pretrained(
"EleutherAI/gpt-j-6b",
device_map={"": 0},
attn_implementation="flash_attention_2",
revision="float16",
torch_dtype=torch.float16,
quantization_config=quantization_config,
)

output_fa_2 = model.generate(**inputs, max_new_tokens=20, do_sample=False)
output_fa_2 = tokenizer.batch_decode(output_fa_2)

self.assertListEqual(expected_outputs, output_fa_2)


@require_torch
class GPTJModelLanguageGenerationTest(unittest.TestCase):
Expand Down
Loading