Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/transformers/models/mamba2/modeling_mamba2.py
Original file line number Diff line number Diff line change
Expand Up @@ -592,8 +592,7 @@ def torch_forward(
L = torch.exp(segment_sum(A))

# Contraction of C and B to get G (attention-weights like)
G_intermediate = C[:, :, :, None, :, :] * B[:, :, None, :, :, :] # shape: (b, c, l, s, h, n)
G = G_intermediate.sum(dim=-1) # shape: (b, c, l, s, h)
G = torch.einsum('bclhn,bcshn->bclsh', C, B)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maxi-nit, I think einsum discard spaces, could we add some here for readability? feels like

G = torch.einsum('b c l h n, b c s h n -> b c l s h', C, B)

is more readable than

G = torch.einsum('bclhn,bcshn->bclsh', C, B)


# Compute M, equivalent to applying attention mask to weights
M_intermediate = G[..., None] * L.permute(0, 2, 3, 4, 1)[..., None]
Expand Down
56 changes: 56 additions & 0 deletions tests/models/mamba2/test_mamba2_consistency.py
Comment thread
SohamWalam11 marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import unittest
import torch
from transformers import Mamba2Config, Mamba2Model


class TestMamba2Consistency(unittest.TestCase):

def setUp(self):
self.config = Mamba2Config(
vocab_size=1000,
hidden_size=64,
num_hidden_layers=1,
expand=2,
num_heads=16,
head_dim=8,
state_size=16,
)
self.input_ids = torch.randint(0, 1000, (1, 4))

def test_training_inference_consistency(self):
model = Mamba2Model(self.config)
torch.manual_seed(42)

model.eval()
with torch.no_grad():
output_inference = model(self.input_ids, use_cache=False)

model.train()
output_training = model(self.input_ids, use_cache=False)

max_diff = torch.max(torch.abs(
output_inference.last_hidden_state - output_training.last_hidden_state.detach()
))

self.assertLess(max_diff.item(), 1e-5,
f"Training/inference outputs differ by {max_diff.item()}")

def test_deterministic_output(self):
model = Mamba2Model(self.config)
model.eval()

torch.manual_seed(42)
with torch.no_grad():
output_1 = model(self.input_ids, use_cache=False)
output_2 = model(self.input_ids, use_cache=False)

max_diff = torch.max(torch.abs(
output_1.last_hidden_state - output_2.last_hidden_state
))

self.assertLess(max_diff.item(), 1e-7,
f"Outputs are not deterministic: {max_diff.item()}")


if __name__ == '__main__':
unittest.main()