From 2e85cdaf15c48f3d876758d165838a4d99f8c269 Mon Sep 17 00:00:00 2001 From: aayup4 Date: Tue, 14 Oct 2025 10:05:02 +0530 Subject: [PATCH 1/5] Optimize Mamba2 memory usage by replacing broadcast with einsum --- .../models/mamba2/modeling_mamba2.py | 3 +- .../models/mamba2/test_mamba2_consistency.py | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/models/mamba2/test_mamba2_consistency.py diff --git a/src/transformers/models/mamba2/modeling_mamba2.py b/src/transformers/models/mamba2/modeling_mamba2.py index bb24e2422d32..5bddbdc1007e 100644 --- a/src/transformers/models/mamba2/modeling_mamba2.py +++ b/src/transformers/models/mamba2/modeling_mamba2.py @@ -592,8 +592,7 @@ def torch_forward( L = torch.exp(segment_sum(A)) # Contraction of C and B to get G (attention-weights like) - G_intermediate = C[:, :, :, None, :, :] * B[:, :, None, :, :, :] # shape: (b, c, l, s, h, n) - G = G_intermediate.sum(dim=-1) # shape: (b, c, l, s, h) + G = torch.einsum('bclhn,bcshn->bclsh', C, B) # Compute M, equivalent to applying attention mask to weights M_intermediate = G[..., None] * L.permute(0, 2, 3, 4, 1)[..., None] diff --git a/tests/models/mamba2/test_mamba2_consistency.py b/tests/models/mamba2/test_mamba2_consistency.py new file mode 100644 index 000000000000..9a151e130a9a --- /dev/null +++ b/tests/models/mamba2/test_mamba2_consistency.py @@ -0,0 +1,56 @@ +import unittest +import torch +from transformers import Mamba2Config, Mamba2Model + + +class TestMamba2Consistency(unittest.TestCase): + + def setUp(self): + self.config = Mamba2Config( + vocab_size=1000, + hidden_size=64, + num_hidden_layers=1, + expand=2, + num_heads=16, + head_dim=8, + state_size=16, + ) + self.input_ids = torch.randint(0, 1000, (1, 4)) + + def test_training_inference_consistency(self): + model = Mamba2Model(self.config) + torch.manual_seed(42) + + model.eval() + with torch.no_grad(): + output_inference = model(self.input_ids, use_cache=False) + + model.train() + output_training = model(self.input_ids, use_cache=False) + + max_diff = torch.max(torch.abs( + output_inference.last_hidden_state - output_training.last_hidden_state.detach() + )) + + self.assertLess(max_diff.item(), 1e-5, + f"Training/inference outputs differ by {max_diff.item()}") + + def test_deterministic_output(self): + model = Mamba2Model(self.config) + model.eval() + + torch.manual_seed(42) + with torch.no_grad(): + output_1 = model(self.input_ids, use_cache=False) + output_2 = model(self.input_ids, use_cache=False) + + max_diff = torch.max(torch.abs( + output_1.last_hidden_state - output_2.last_hidden_state + )) + + self.assertLess(max_diff.item(), 1e-7, + f"Outputs are not deterministic: {max_diff.item()}") + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 642c406816bea108f1bfa1928c0ff26c7a168ef9 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 18 Nov 2025 16:01:02 +0000 Subject: [PATCH 2/5] engage full einsum mode --- .../models/mamba2/modeling_mamba2.py | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/mamba2/modeling_mamba2.py b/src/transformers/models/mamba2/modeling_mamba2.py index f12260c842d1..b3c6b4f3e26c 100644 --- a/src/transformers/models/mamba2/modeling_mamba2.py +++ b/src/transformers/models/mamba2/modeling_mamba2.py @@ -590,22 +590,12 @@ def torch_forward( # 1. Compute the output for each intra-chunk (diagonal blocks) # This is the analog of a causal mask L = torch.exp(segment_sum(A)) - - # Contraction of C and B to get G (attention-weights like) - G = torch.einsum('bclhn,bcshn->bclsh', C, B) - - # Compute M, equivalent to applying attention mask to weights - M_intermediate = G[..., None] * L.permute(0, 2, 3, 4, 1)[..., None] - M = M_intermediate.sum(dim=-1) - - # Compute Y_diag (apply to values) - Y_diag = (M[..., None] * hidden_states[:, :, None]).sum(dim=3) + Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, X) # 2. Compute the state for each intra-chunk # (right term of low-rank factorization of off-diagonal blocks; B terms) decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum) - B_decay = B * decay_states.permute(0, -2, -1, 1)[..., None] - states = (B_decay[..., None, :] * hidden_states[..., None]).sum(dim=2) + states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, X) # 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at chunk boundaries # (middle term of factorization of off-diag blocks; A terms) @@ -615,16 +605,13 @@ def torch_forward( previous_states = torch.zeros_like(states[:, :1]) states = torch.cat([previous_states, states], dim=1) decay_chunk = torch.exp(segment_sum(nn.functional.pad(A_cumsum[:, :, :, -1], (1, 0)))) - decay_chunk = decay_chunk.transpose(1, 3) - new_states = (decay_chunk[..., None, None] * states[:, :, None, ...]).sum(dim=1) + new_states = torch.einsum("bhzc,bchpn->bzhpn", decay_chunk, states) states, ssm_state = new_states[:, :-1], new_states[:, -1] # 4. Compute state -> output conversion per chunk # (left term of low-rank factorization of off-diagonal blocks; C terms) state_decay_out = torch.exp(A_cumsum) - C_times_states = (C[..., None, :] * states[:, :, None, ...]) - state_decay_out_permuted = state_decay_out.permute(0, 2, 3, 1) - Y_off = (C_times_states.sum(-1) * state_decay_out_permuted[..., None]) + Y_off = torch.einsum('bclhn,bchpn,bhcl->bclhp', C, states, state_decay_out) # Add output of intra-chunk and inter-chunk terms (diagonal and off-diagonal blocks) y = Y_diag + Y_off From 40ff4f0990feb740b39a7b8c6c4217b207d6d369 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 18 Nov 2025 16:02:00 +0000 Subject: [PATCH 3/5] remove unnecessary test --- .../models/mamba2/test_mamba2_consistency.py | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 tests/models/mamba2/test_mamba2_consistency.py diff --git a/tests/models/mamba2/test_mamba2_consistency.py b/tests/models/mamba2/test_mamba2_consistency.py deleted file mode 100644 index 9a151e130a9a..000000000000 --- a/tests/models/mamba2/test_mamba2_consistency.py +++ /dev/null @@ -1,56 +0,0 @@ -import unittest -import torch -from transformers import Mamba2Config, Mamba2Model - - -class TestMamba2Consistency(unittest.TestCase): - - def setUp(self): - self.config = Mamba2Config( - vocab_size=1000, - hidden_size=64, - num_hidden_layers=1, - expand=2, - num_heads=16, - head_dim=8, - state_size=16, - ) - self.input_ids = torch.randint(0, 1000, (1, 4)) - - def test_training_inference_consistency(self): - model = Mamba2Model(self.config) - torch.manual_seed(42) - - model.eval() - with torch.no_grad(): - output_inference = model(self.input_ids, use_cache=False) - - model.train() - output_training = model(self.input_ids, use_cache=False) - - max_diff = torch.max(torch.abs( - output_inference.last_hidden_state - output_training.last_hidden_state.detach() - )) - - self.assertLess(max_diff.item(), 1e-5, - f"Training/inference outputs differ by {max_diff.item()}") - - def test_deterministic_output(self): - model = Mamba2Model(self.config) - model.eval() - - torch.manual_seed(42) - with torch.no_grad(): - output_1 = model(self.input_ids, use_cache=False) - output_2 = model(self.input_ids, use_cache=False) - - max_diff = torch.max(torch.abs( - output_1.last_hidden_state - output_2.last_hidden_state - )) - - self.assertLess(max_diff.item(), 1e-7, - f"Outputs are not deterministic: {max_diff.item()}") - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file From 2005678506ee48d9a4a7d4b57f286bcf7b41a6c8 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 18 Nov 2025 16:06:21 +0000 Subject: [PATCH 4/5] X -> hidden_states --- src/transformers/models/mamba2/modeling_mamba2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/mamba2/modeling_mamba2.py b/src/transformers/models/mamba2/modeling_mamba2.py index b3c6b4f3e26c..f48c99fc2816 100644 --- a/src/transformers/models/mamba2/modeling_mamba2.py +++ b/src/transformers/models/mamba2/modeling_mamba2.py @@ -590,12 +590,12 @@ def torch_forward( # 1. Compute the output for each intra-chunk (diagonal blocks) # This is the analog of a causal mask L = torch.exp(segment_sum(A)) - Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, X) + Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, hidden_states) # 2. Compute the state for each intra-chunk # (right term of low-rank factorization of off-diagonal blocks; B terms) decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum) - states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, X) + states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, hidden_states) # 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at chunk boundaries # (middle term of factorization of off-diag blocks; A terms) From 21eea32bf90314a227f37302b6fb8d5288569457 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 18 Nov 2025 16:48:47 +0000 Subject: [PATCH 5/5] Spaces for readability --- src/transformers/models/mamba2/modeling_mamba2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/mamba2/modeling_mamba2.py b/src/transformers/models/mamba2/modeling_mamba2.py index f48c99fc2816..69dfc1302837 100644 --- a/src/transformers/models/mamba2/modeling_mamba2.py +++ b/src/transformers/models/mamba2/modeling_mamba2.py @@ -590,12 +590,12 @@ def torch_forward( # 1. Compute the output for each intra-chunk (diagonal blocks) # This is the analog of a causal mask L = torch.exp(segment_sum(A)) - Y_diag = torch.einsum("bclhn,bcshn,bhcls,bcshp->bclhp", C, B, L, hidden_states) + Y_diag = torch.einsum("bclhn, bcshn, bhcls, bcshp -> bclhp", C, B, L, hidden_states) # 2. Compute the state for each intra-chunk # (right term of low-rank factorization of off-diagonal blocks; B terms) decay_states = torch.exp(A_cumsum[:, :, :, -1:] - A_cumsum) - states = torch.einsum("bclhn,bhcl,bclhp->bchpn", B, decay_states, hidden_states) + states = torch.einsum("bclhn, bhcl, bclhp -> bchpn", B, decay_states, hidden_states) # 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at chunk boundaries # (middle term of factorization of off-diag blocks; A terms) @@ -605,13 +605,13 @@ def torch_forward( previous_states = torch.zeros_like(states[:, :1]) states = torch.cat([previous_states, states], dim=1) decay_chunk = torch.exp(segment_sum(nn.functional.pad(A_cumsum[:, :, :, -1], (1, 0)))) - new_states = torch.einsum("bhzc,bchpn->bzhpn", decay_chunk, states) + new_states = torch.einsum("bhzc, bchpn -> bzhpn", decay_chunk, states) states, ssm_state = new_states[:, :-1], new_states[:, -1] # 4. Compute state -> output conversion per chunk # (left term of low-rank factorization of off-diagonal blocks; C terms) state_decay_out = torch.exp(A_cumsum) - Y_off = torch.einsum('bclhn,bchpn,bhcl->bclhp', C, states, state_decay_out) + Y_off = torch.einsum('bclhn, bchpn, bhcl -> bclhp', C, states, state_decay_out) # Add output of intra-chunk and inter-chunk terms (diagonal and off-diagonal blocks) y = Y_diag + Y_off