diff --git a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py index fec83f1b18c3..9b0ae6f7f8da 100644 --- a/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py +++ b/src/transformers/models/qwen2_5_omni/modeling_qwen2_5_omni.py @@ -1586,7 +1586,8 @@ def forward( text_position_ids = position_ids[0] position_ids = position_ids[1:] else: - text_position_ids = position_ids[0] + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -2161,7 +2162,8 @@ def forward( text_position_ids = position_ids[0] position_ids = position_ids[1:] else: - text_position_ids = position_ids[0] + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): diff --git a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py index 6f050d6d1ac6..8a04e8116eb5 100644 --- a/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py @@ -865,7 +865,8 @@ def forward( text_position_ids = position_ids[0] position_ids = position_ids[1:] else: - text_position_ids = position_ids[0] + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -1308,7 +1309,7 @@ def forward( else: delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device) delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=1) - position_ids += delta.to(position_ids.device) + position_ids = position_ids + delta.to(position_ids.device) outputs = self.language_model( input_ids=None, @@ -1573,17 +1574,16 @@ def prepare_inputs_for_generation( self.model.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids elif "position_ids" in model_inputs: - position_ids = model_inputs["position_ids"][None, ...] - delta = self.model.rope_deltas - delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0) + batch_size, seq_length = model_inputs["position_ids"].shape + device = model_inputs["position_ids"].device + position_ids = torch.arange(seq_length, device=device) + position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1) + delta = cache_position[0] + self.model.rope_deltas + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) vision_positions = position_ids + delta.expand_as(position_ids) - vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1) # Concatenate "text + vision" positions into [4, bs, seq-len] - if "position_ids" not in model_inputs: - text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :] - else: - text_positions = model_inputs["position_ids"][None, ...] + text_positions = model_inputs["position_ids"][None, ...] model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0) if cache_position[0] != 0: diff --git a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py index 7b47fdb199d6..90ec79edf0ea 100644 --- a/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py +++ b/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py @@ -622,7 +622,7 @@ def forward( else: delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device) delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=1) - position_ids += delta.to(position_ids.device) + position_ids = position_ids + delta.to(position_ids.device) outputs = self.language_model( input_ids=None, @@ -818,17 +818,16 @@ def prepare_inputs_for_generation( self.model.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids elif "position_ids" in model_inputs: - position_ids = model_inputs["position_ids"][None, ...] - delta = self.model.rope_deltas - delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0) + batch_size, seq_length = model_inputs["position_ids"].shape + device = model_inputs["position_ids"].device + position_ids = torch.arange(seq_length, device=device) + position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1) + delta = cache_position[0] + self.model.rope_deltas + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) vision_positions = position_ids + delta.expand_as(position_ids) - vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1) # Concatenate "text + vision" positions into [4, bs, seq-len] - if "position_ids" not in model_inputs: - text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :] - else: - text_positions = model_inputs["position_ids"][None, ...] + text_positions = model_inputs["position_ids"][None, ...] model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0) if cache_position[0] != 0: diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index a7e458ac9b10..745829856db0 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -838,7 +838,8 @@ def forward( text_position_ids = position_ids[0] position_ids = position_ids[1:] else: - text_position_ids = position_ids[0] + # If inputs are not packed (usual 3D positions), do not prepare mask from position_ids + text_position_ids = None # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -1228,7 +1229,7 @@ def forward( else: delta = torch.zeros((batch_size, seq_length), device=inputs_embeds.device) delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) - position_ids += delta.to(position_ids.device) + position_ids = position_ids + delta.to(position_ids.device) outputs = self.language_model( input_ids=None, @@ -1458,17 +1459,16 @@ def prepare_inputs_for_generation( self.model.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids elif "position_ids" in model_inputs: - position_ids = model_inputs["position_ids"][None, ...] - delta = self.model.rope_deltas - delta = delta.repeat_interleave(position_ids.shape[1] // delta.shape[0], dim=0) + batch_size, seq_length = model_inputs["position_ids"].shape + device = model_inputs["position_ids"].device + position_ids = torch.arange(seq_length, device=device) + position_ids = position_ids.view(1, 1, -1).expand(3, batch_size, -1) + delta = cache_position[0] + self.model.rope_deltas + delta = delta.repeat_interleave(batch_size // delta.shape[0], dim=0) vision_positions = position_ids + delta.expand_as(position_ids) - vision_positions = vision_positions.expand(3, vision_positions.shape[1], -1) # Concatenate "text + vision" positions into [4, bs, seq-len] - if "position_ids" not in model_inputs: - text_positions = torch.arange(input_ids, device=input_ids.device)[None, None, :] - else: - text_positions = model_inputs["position_ids"][None, ...] + text_positions = model_inputs["position_ids"][None, ...] model_inputs["position_ids"] = torch.cat([text_positions, vision_positions], dim=0) if model_inputs["cache_position"][0] != 0: diff --git a/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py b/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py index 8be3deffd516..446163b7794f 100644 --- a/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py +++ b/tests/models/qwen2_5_vl/test_modeling_qwen2_5_vl.py @@ -636,8 +636,8 @@ def test_small_model_integration_test_batch_flashatt2(self): output = model.generate(**inputs, max_new_tokens=30) EXPECTED_DECODED_TEXT = [ - "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets", - "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets", + "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in'", + "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in'", ] self.assertEqual( @@ -673,9 +673,9 @@ def test_small_model_integration_test_batch_wo_image_flashatt2(self): output = model.generate(**inputs, max_new_tokens=30) EXPECTED_DECODED_TEXT = [ - "system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and intelligent nature, making them popular pets", - "system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\nI am Qwen, a large language model created by Alibaba Cloud. I am designed to answer a wide range of questions and provide information on various topics", - ] + 'system\nYou are a helpful assistant.\nuser\nWhat kind of dog is this?\nassistant\nThe dog in the picture appears to be a Labrador Retriever. Labradors are known for their friendly and energetic nature, which is evident in', + "system\nYou are a helpful assistant.\nuser\nWho are you?\nassistant\n�\n\n addCriterion\nI'm sorry, but I don't understand your question. Could you please provide more context or clarify what you're asking", + ] # fmt: skip self.assertEqual( self.processor.batch_decode(output, skip_special_tokens=True), diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 21cb0fcf178e..0b3ad0d6539d 100755 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -4767,6 +4767,15 @@ def update_config_headdim(config, requested_dim): head_dim = head_dim if head_dim is not None else config.hidden_size // config.num_attention_heads config.hidden_size *= max(requested_dim // head_dim, 1) + # Some models use 3D RoPE where the sum of RoPE sections has to be equal to head dim + if ( + getattr(config, "rope_scaling", None) is not None + and config.rope_scaling.get("mrope_section") is not None + ): + mrope_section = config.rope_scaling["mrope_section"] + mutiplier = max(requested_dim // head_dim, 1) + config.rope_scaling = {"type": "default", "mrope_section": [i * mutiplier for i in mrope_section]} + if ( getattr(config, "decoder_hidden_size", None) is not None and getattr(config, "decoder_num_attention_heads", None) is not None