Skip to content
Closed
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 9 additions & 17 deletions src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,6 @@ def extra_repr(self):
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"


def rotate_half(x):
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)


def apply_rotary_pos_emb(
x: torch.Tensor,
cos: torch.Tensor,
Expand All @@ -93,22 +86,22 @@ def apply_rotary_pos_emb(
Returns:
`torch.Tensor`: Tensor with rotary embeddings applied, same shape as input.
"""
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)

# Split-half (NeoX/Llama style): (x[:d/2], x[d/2:])
# This matches llama's apply_rotary_pos_emb logic.
x_rotated = (x * cos) + (rotate_half(x) * sin)
return x_rotated
# Interleaved (GPT-J style): (x[0], x[1]), (x[2], x[3]), ...
# RotaryEmbedding outputs cos/sin with repeated halves for NeoX compatibility,
# while interleaved rotation expects [.., D/2] frequencies.
cos = cos[..., : x.shape[-1] // 2].unsqueeze(unsqueeze_dim)
sin = sin[..., : x.shape[-1] // 2].unsqueeze(unsqueeze_dim)
x1 = x[..., ::2]
x2 = x[..., 1::2]
return torch.stack((x1 * cos - x2 * sin, x2 * cos + x1 * sin), dim=-1).flatten(-2)


class GlmMoeDsaIndexer(nn.Module):
"""
Dynamic Sparse Attention (DSA) indexer for selecting top-k tokens.

The Indexer has its own lightweight projections (wq_b, wk) separate from the
main MLA attention. It uses non-interleaved (NeoX/Llama) RoPE, unlike the main attention
which uses interleaved RoPE.
main MLA attention.

**Cache strategy**: The Indexer manages its own key cache (`_cached_keys`) separately
from the DynamicCache used by MLA attention, since DynamicCache is sized for exactly
Expand Down Expand Up @@ -216,7 +209,6 @@ def forward(

# q·k^T per head: [B, S, H, D] @ [B, T, D]^T → [B, S, H, T]
scores = torch.einsum("bshd,btd->bsht", q.float(), k_cached.float()) * self.softmax_scale
scores = F.relu(scores)
# Weight per head and sum across heads → [B, S, T]
index_scores = torch.einsum("bsht,bsh->bst", scores, weights)

Expand Down
20 changes: 9 additions & 11 deletions src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from ...configuration_utils import PreTrainedConfig
from ...modeling_flash_attention_utils import FlashAttentionKwargs
from ...modeling_utils import ALL_ATTENTION_FUNCTIONS
from ...models.llama.modeling_llama import rotate_half
from ...processing_utils import Unpack
from ...utils import auto_docstring, logging
from ...utils.generic import is_flash_attention_requested
Expand Down Expand Up @@ -65,13 +64,14 @@ def apply_rotary_pos_emb(
Returns:
`torch.Tensor`: Tensor with rotary embeddings applied, same shape as input.
"""
cos = cos.unsqueeze(unsqueeze_dim)
sin = sin.unsqueeze(unsqueeze_dim)

# Split-half (NeoX/Llama style): (x[:d/2], x[d/2:])
# This matches llama's apply_rotary_pos_emb logic.
x_rotated = (x * cos) + (rotate_half(x) * sin)
return x_rotated
# Interleaved (GPT-J style): (x[0], x[1]), (x[2], x[3]), ...
# RotaryEmbedding outputs cos/sin with repeated halves for NeoX compatibility,
# while interleaved rotation expects [.., D/2] frequencies.
cos = cos[..., : x.shape[-1] // 2].unsqueeze(unsqueeze_dim)
sin = sin[..., : x.shape[-1] // 2].unsqueeze(unsqueeze_dim)
x1 = x[..., ::2]
x2 = x[..., 1::2]
return torch.stack((x1 * cos - x2 * sin, x2 * cos + x1 * sin), dim=-1).flatten(-2)


@auto_docstring(checkpoint="zai-org/GLM-5")
Expand Down Expand Up @@ -154,8 +154,7 @@ class GlmMoeDsaIndexer(nn.Module):
Dynamic Sparse Attention (DSA) indexer for selecting top-k tokens.

The Indexer has its own lightweight projections (wq_b, wk) separate from the
main MLA attention. It uses non-interleaved (NeoX/Llama) RoPE, unlike the main attention
which uses interleaved RoPE.
main MLA attention.

**Cache strategy**: The Indexer manages its own key cache (`_cached_keys`) separately
from the DynamicCache used by MLA attention, since DynamicCache is sized for exactly
Expand Down Expand Up @@ -263,7 +262,6 @@ def forward(

# q·k^T per head: [B, S, H, D] @ [B, T, D]^T → [B, S, H, T]
scores = torch.einsum("bshd,btd->bsht", q.float(), k_cached.float()) * self.softmax_scale
scores = F.relu(scores)
# Weight per head and sum across heads → [B, S, T]
index_scores = torch.einsum("bsht,bsh->bst", scores, weights)

Expand Down
Loading