Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py
Original file line number Diff line number Diff line change
Expand Up @@ -1586,7 +1586,7 @@ def forward(
text_position_ids = position_ids[0]
position_ids = position_ids[1:]
else:
text_position_ids = position_ids[0]
text_position_ids = None
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved

# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
Expand Down Expand Up @@ -2161,7 +2161,7 @@ def forward(
text_position_ids = position_ids[0]
position_ids = position_ids[1:]
else:
text_position_ids = position_ids[0]
text_position_ids = None

# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
Expand Down
19 changes: 9 additions & 10 deletions src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py
Original file line number Diff line number Diff line change
Expand Up @@ -865,7 +865,7 @@ def forward(
text_position_ids = position_ids[0]
position_ids = position_ids[1:]
else:
text_position_ids = position_ids[0]
text_position_ids = None

# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
Expand Down Expand Up @@ -1308,7 +1308,7 @@ def forward(
else:
delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device)
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=1)
position_ids += delta.to(position_ids.device)
position_ids = position_ids + delta.to(position_ids.device)
Comment thread
vasqu marked this conversation as resolved.
Resolved

outputs = self.language_model(
input_ids=None,
Expand Down Expand Up @@ -1573,17 +1573,16 @@ def prepare_inputs_for_generation(
self.model.rope_deltas = rope_deltas
# then use the prev pre-calculated rope-deltas to get the correct position ids
elif "position_ids" in model_inputs:
position_ids = model_inputs["position_ids"][None, ...]
Comment thread
vasqu marked this conversation as resolved.
Resolved
delta = self.model.rope_deltas
delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0)
batch_size, seq_length = model_inputs["position_ids"].shape
device = model_inputs["position_ids"].device
position_ids = torch.arange(seq_length, device=device)
position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1)
delta = cache_position[0] + self.model.rope_deltas
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
vision_positions = position_ids + delta.expand_as(position_ids)
vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1)

# Concatenate "text + vision" positions into [4, bs, seq-len]
if "position_ids" not in model_inputs:
text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :]
else:
text_positions = model_inputs["position_ids"][None, ...]
text_positions = model_inputs["position_ids"][None, ...]
model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0)

if cache_position[0] != 0:
Expand Down
17 changes: 8 additions & 9 deletions src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@ def forward(
else:
delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device)
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=1)
position_ids += delta.to(position_ids.device)
position_ids = position_ids + delta.to(position_ids.device)

outputs = self.language_model(
input_ids=None,
Expand Down Expand Up @@ -818,17 +818,16 @@ def prepare_inputs_for_generation(
self.model.rope_deltas = rope_deltas
# then use the prev pre-calculated rope-deltas to get the correct position ids
elif "position_ids" in model_inputs:
position_ids = model_inputs["position_ids"][None, ...]
delta = self.model.rope_deltas
delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0)
batch_size, seq_length = model_inputs["position_ids"].shape
device = model_inputs["position_ids"].device
position_ids = torch.arange(seq_length, device=device)
position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1)
delta = cache_position[0] + self.model.rope_deltas
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
vision_positions = position_ids + delta.expand_as(position_ids)
vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1)

# Concatenate "text + vision" positions into [4, bs, seq-len]
if "position_ids" not in model_inputs:
text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :]
else:
text_positions = model_inputs["position_ids"][None, ...]
text_positions = model_inputs["position_ids"][None, ...]
Comment thread
vasqu marked this conversation as resolved.
Resolved
model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0)

if cache_position[0] != 0:
Expand Down
19 changes: 9 additions & 10 deletions src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,7 @@ def forward(
text_position_ids = position_ids[0]
position_ids = position_ids[1:]
else:
text_position_ids = position_ids[0]
text_position_ids = None

# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
Expand Down Expand Up @@ -1228,7 +1228,7 @@ def forward(
else:
delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device)
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
position_ids += delta.to(position_ids.device)
position_ids = position_ids + delta.to(position_ids.device)

outputs = self.language_model(
input_ids=None,
Expand Down Expand Up @@ -1458,17 +1458,16 @@ def prepare_inputs_for_generation(
self.model.rope_deltas = rope_deltas
# then use the prev pre-calculated rope-deltas to get the correct position ids
elif "position_ids" in model_inputs:
position_ids = model_inputs["position_ids"][None, ...]
delta = self.model.rope_deltas
delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0)
batch_size, seq_length = model_inputs["position_ids"].shape
device = model_inputs["position_ids"].device
position_ids = torch.arange(seq_length, device=device)
position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1)
delta = cache_position[0] + self.model.rope_deltas
delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0)
vision_positions = position_ids + delta.expand_as(position_ids)
vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1)

# Concatenate "text + vision" positions into [4, bs, seq-len]
if "position_ids" not in model_inputs:
text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :]
else:
text_positions = model_inputs["position_ids"][None, ...]
text_positions = model_inputs["position_ids"][None, ...]
model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0)

if model_inputs["cache_position"][0] != 0:
Expand Down
10 changes: 5 additions & 5 deletions tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py
Original file line number Diff line number Diff line change
Expand Up @@ -636,8 +636,8 @@ def test_small_model_integration_test_batch_flashatt2(self):
output = model.generate(**inputs, max_new_tokens=30)

EXPECTED_DECODED_TEXT = [
"system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets",
"system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets",
"system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in'",
"system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in'",
]

self.assertEqual(
Expand Down Expand Up @@ -673,9 +673,9 @@ def test_small_model_integration_test_batch_wo_image_flashatt2(self):
output = model.generate(**inputs, max_new_tokens=30)

EXPECTED_DECODED_TEXT = [
"system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets",
"system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\nI am Qwen, a large language model created by Alibaba Cloud. I am designed to answer a wide range of questions and provide information on various topics",
]
'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in',
"system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\n�\n\n addCriterion\nI'm sorry, but I don't understand your question. Could you please provide more context or clarify what you're asking",
] # fmt: skip

self.assertEqual(
self.processor.batch_decode(output, skip_special_tokens=True),
Expand Down
9 changes: 9 additions & 0 deletions tests/test_modeling_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -4767,6 +4767,15 @@ def update_config_headdim(config, requested_dim):
head_dim = head_dim if head_dim is not None else config.hidden_size // config.num_attention_heads
config.hidden_size *= max(requested_dim // head_dim, 1)

# Some models use 3D RoPE where the sum of RoPE sections has to be equal to head dim
if (
getattr(config, "rope_scaling", None) is not None
and config.rope_scaling.get("mrope_section") is not None
):
mrope_section = config.rope_scaling["mrope_section"]
mutiplier = max(requested_dim // head_dim, 1)
config.rope_scaling = {"type": "default", "mrope_section": [i * mutiplier for i in mrope_section]}
Comment thread
zucchini-nlp marked this conversation as resolved.
Resolved

if (
getattr(config, "decoder_hidden_size", None) is not None
and getattr(config, "decoder_num_attention_heads", None) is not None
Expand Down