Repository navigation
MRoPE continued #48594
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
MRoPE continued #48594
Changes from 4 commits
6e272c5
4b32510
c8cd232
15c97ef
bf3e072
cde3c99
50d0861
82461b0
fbe398f
1d880c4
9abb901
d200bae
52e3885
31f0ecf
d39b672
3c1af41
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -740,6 +740,41 @@ class RotaryEmbeddingConfigMixin: | |
| default_rope_type = "default" # override only for axial models | ||
| ignore_keys_at_rope_validation = set() | ||
|
|
||
| @property | ||
| def has_per_layer_rope(self) -> bool: | ||
| if self.rope_parameters is None or getattr(self, "layer_types", None) is None: | ||
| return False | ||
| if set(self.rope_parameters.keys()).isdisjoint(self.layer_types): | ||
| return False | ||
| return True | ||
|
|
||
| @property | ||
| def num_multimodal_rope_axis(self, layer_type: str | None = None) -> int | None: | ||
| if ( | ||
| self.rope_parameters is None | ||
| or (not self.has_per_layer_rope and "mrope_section" not in self.rope_parameters) | ||
| or ( | ||
| self.has_per_layer_rope | ||
| and not any("mrope_section" in sub_dict for sub_dict in self.rope_parameters.values()) | ||
| ) | ||
| ): | ||
| return None | ||
|
|
||
| if not self.has_per_layer_rope: | ||
| mrope_section = self.rope_parameters.get("mrope_section") | ||
| else: | ||
| mrope_sections = [ | ||
| sub_dict["mrope_section"] | ||
| for sub_dict in self.rope_parameters.values() | ||
| if sub_dict and "mrope_section" in sub_dict | ||
| ] | ||
| if len({tuple(sections) for sections in mrope_sections}) > 1 and layer_type is None: | ||
| raise ValueError( | ||
| "Model has different `mrope_section` per layer type, please pass `layer_type` argument" | ||
| ) | ||
| mrope_section = mrope_sections[0] | ||
| return len(mrope_section) | ||
|
|
||
| def convert_rope_params_to_dict(self, **kwargs): | ||
| rope_scaling = kwargs.pop("rope_scaling", None) | ||
| self.rope_parameters = rope_scaling or self.rope_parameters | ||
|
|
@@ -786,7 +821,7 @@ def standardize_rope_params(self): | |
| rope_parameters.setdefault("rope_type", rope_parameters.get("type", "default")) | ||
| rope_parameters.setdefault("rope_theta", rope_theta) | ||
| if partial_rotary_factor is not None: | ||
| rope_parameters["partial_rotary_factor"] = partial_rotary_factor | ||
| rope_parameters.setdefault("partial_rotary_factor", partial_rotary_factor) | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. don't force override if the dict has a value set, mirror
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. uh oh do we maybe have a test for that |
||
|
|
||
| # Force set the default type to model's expected `default_rope`. For most models it's a no-op | ||
| # used only to keep BC with old ckpt that require axial rope type | ||
|
|
@@ -812,7 +847,7 @@ def standardize_rope_params(self): | |
| rope_parameters[layer_type].setdefault("rope_type", rope_parameters[layer_type].get("type", "default")) | ||
| rope_parameters[layer_type].setdefault("rope_theta", rope_theta) | ||
| if partial_rotary_factor is not None: | ||
| rope_parameters[layer_type]["partial_rotary_factor"] = partial_rotary_factor | ||
| rope_parameters[layer_type].setdefault("partial_rotary_factor", partial_rotary_factor) | ||
|
|
||
| if rope_parameters[layer_type]["rope_type"] in ["llama3", "yarn", "longrope"]: | ||
| self.rope_parameters[layer_type].setdefault( | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -46,7 +46,6 @@ | |
| from ...utils.generic import ( | ||
| get_max_seqlen, | ||
| is_flash_attention_requested, | ||
| maybe_autocast, | ||
| merge_with_config_defaults, | ||
| ) | ||
| from ...utils.output_capturing import capture_outputs | ||
|
|
@@ -62,7 +61,6 @@ | |
| LlamaDecoderLayer, | ||
| LlamaModel, | ||
| LlamaRMSNorm, | ||
| LlamaRotaryEmbedding, | ||
| eager_attention_forward, | ||
| ) | ||
| from ..qwen2_vl.modeling_qwen2_vl import ( | ||
|
|
@@ -72,6 +70,7 @@ | |
| TransformersKwargs, | ||
| ) | ||
| from ..qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeVisionPatchMerger | ||
| from ..qwen3_vl.modeling_qwen3_vl import Qwen3VLTextRotaryEmbedding | ||
| from ..qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor | ||
| from ..siglip2.configuration_siglip2 import Siglip2VisionConfig | ||
| from ..siglip2.modeling_siglip2 import ( | ||
|
|
@@ -215,41 +214,8 @@ def validate_architecture(self): | |
| raise TypeError("`vision_config` must be a `Cosmos3EdgeVisionConfig` or a dictionary.") | ||
|
|
||
|
|
||
| class Cosmos3EdgeTextRotaryEmbedding(LlamaRotaryEmbedding): | ||
| """Interleaved M-RoPE used for Cosmos3 Edge text and visual tokens.""" | ||
|
|
||
| def compute_default_rope_parameters( | ||
| config: Cosmos3EdgeTextConfig, device=None, **kwargs | ||
| ) -> tuple[torch.Tensor, float]: | ||
| """Construct an axis-aware inverse-frequency matrix for interleaved temporal, height, and width RoPE.""" | ||
| base = config.rope_parameters["rope_theta"] | ||
| dim = config.head_dim | ||
| inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) | ||
|
|
||
| indices = torch.arange(inv_freq.shape[0]) | ||
| mrope_section = config.rope_parameters["mrope_section"] | ||
| height_mask = (indices % 3 == 1) & (indices < mrope_section[1] * 3) | ||
| width_mask = (indices % 3 == 2) & (indices < mrope_section[2] * 3) | ||
| temporal_mask = ~(height_mask | width_mask) | ||
| inv_freq = torch.stack( | ||
| ( | ||
| inv_freq * temporal_mask, | ||
| inv_freq * height_mask, | ||
| inv_freq * width_mask, | ||
| ) | ||
| ) | ||
| return inv_freq.to(device), 1.0 | ||
|
|
||
| @torch.no_grad() | ||
| def forward(self, x, position_ids): | ||
| position_ids = position_ids.permute(1, 2, 0).float() | ||
| device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" | ||
| with maybe_autocast(device_type=device_type, enabled=False): | ||
| freqs = position_ids.float() @ self.inv_freq.float().to(x.device) | ||
| emb = torch.cat((freqs, freqs), dim=-1) | ||
| cos = emb.cos() * self.attention_scaling | ||
| sin = emb.sin() * self.attention_scaling | ||
| return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) | ||
| class Cosmos3EdgeTextRotaryEmbedding(Qwen3VLTextRotaryEmbedding): | ||
| pass | ||
|
Comment on lines
+217
to
+218
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. was really identical, we gotta be careful when reviewing models so contribs don't re-invent the wheel
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sadly will and can happen at times. This is why it's more important than ever to have a more uniform recipe 🫡 |
||
|
|
||
|
|
||
| class Cosmos3EdgeTextAttention(LlamaAttention): | ||
|
|
||
|
zucchini-nlp marked this conversation as resolved.
Resolved
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,22 +37,21 @@ | |
| logging, | ||
| torch_compilable_check, | ||
| ) | ||
| from ...utils.deprecation import deprecate_kwarg | ||
| from ...utils.generic import ( | ||
| accepts_precomputed_kwargs, | ||
| maybe_autocast, | ||
| merge_with_config_defaults, | ||
| ) | ||
| from ...utils.output_capturing import capture_outputs | ||
| from ...vision_utils import get_vision_attention_seqlens, get_vision_position_ids | ||
| from ..glm4.modeling_glm4 import Glm4MLP, Glm4RMSNorm, Glm4RotaryEmbedding, eager_attention_forward | ||
| from ..glm4.modeling_glm4 import Glm4MLP, Glm4RMSNorm, eager_attention_forward | ||
| from ..qwen2_5_vl.modeling_qwen2_5_vl import ( | ||
| Qwen2_5_VisionPatchEmbed, | ||
| Qwen2_5_VLCausalLMOutputWithPast, | ||
| Qwen2_5_VLForConditionalGeneration, | ||
| Qwen2_5_VLMLP, | ||
| Qwen2_5_VLModelOutputWithPast, | ||
| Qwen2_5_VLPreTrainedModel, | ||
| Qwen2_5_VLRotaryEmbedding, | ||
| Qwen2_5_VLTextModel, | ||
| Qwen2_5_VLVisionAttention, | ||
| Qwen2_5_VLVisionBlock, | ||
|
|
@@ -368,33 +367,17 @@ def __init__(self, config) -> None: | |
| self.mlp = Glm4VisionMlp(config, bias=False) | ||
|
|
||
|
|
||
| class Glm4vTextRotaryEmbedding(Glm4RotaryEmbedding): | ||
| @deprecate_kwarg("device", version="5.18") | ||
| class Glm4vTextRotaryEmbedding(Qwen2_5_VLRotaryEmbedding): | ||
| def __init__(self, config: Glm4vTextConfig, device=None): | ||
| super().__init__() | ||
| self.mrope_section = config.rope_parameters.get("mrope_section", [8, 12, 12]) | ||
|
|
||
| def forward(self, x, position_ids): | ||
| # In contrast to other models, GLM-V has different position ids for the grids | ||
| # So we expand the inv_freq to shape (3, ...) | ||
| inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1) | ||
| position_ids_expanded = position_ids[:, :, None, :].float() # shape (3, bs, 1, positions) | ||
|
|
||
| device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" | ||
| with maybe_autocast(device_type=device_type, enabled=False): # Force float32 | ||
| freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3) | ||
| freqs = self.apply_mrope(freqs, self.mrope_section) | ||
| emb = torch.cat((freqs, freqs), dim=-1) | ||
| cos = emb.cos() * self.attention_scaling | ||
| sin = emb.sin() * self.attention_scaling | ||
|
|
||
| return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) | ||
|
|
||
| def apply_mrope(self, freqs, mrope_section): | ||
| section = mrope_section | ||
| chunks = freqs.split(section, dim=-1) | ||
| result = torch.cat([chunk[i % 3] for i, chunk in enumerate(chunks)], dim=-1) | ||
| return result | ||
| def recomposition_frequencies(self, freq): | ||
| """ | ||
| Recompose the frequencies into the final spatial layout used per each grid. | ||
| """ | ||
| freq = torch.cat([m[i % 3] for i, m in enumerate(freq.split(self.mrope_section, dim=-1))], dim=-1) | ||
| return freq.repeat_interleave(2, dim=-1) | ||
|
|
||
|
|
||
| def rotate_half_llm(x): | ||
|
|
@@ -425,10 +408,6 @@ def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): | |
| cos = cos.unsqueeze(unsqueeze_dim) | ||
| sin = sin.unsqueeze(unsqueeze_dim) | ||
|
|
||
| # Interleave them instead of usual shape | ||
| cos = cos[..., : cos.shape[-1] // 2].repeat_interleave(2, dim=-1) | ||
| sin = sin[..., : sin.shape[-1] // 2].repeat_interleave(2, dim=-1) | ||
|
|
||
|
Comment on lines
-428
to
-431
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is so weird 😆
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Argh I swear we can get there tho! I will maybe check whether I can work on the kernels side for this
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe open an issue for me :D |
||
| # Keep half or full tensor for later concatenation | ||
| rotary_dim = cos.shape[-1] | ||
| q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:] | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.