Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
3f9eeb8
modify mamba cache
zhenglongjiepheonix Jun 4, 2024
d6413fe
set up cache
zhenglongjiepheonix Jun 5, 2024
054d7cf
add test
zhenglongjiepheonix Jun 6, 2024
93fa82f
fix conflict
zhenglongjiepheonix Jun 6, 2024
48e0ff0
[run-slow] mamba
zhenglongjiepheonix Jun 6, 2024
1e6f0e9
[run-slow] mamba
zhenglongjiepheonix Jun 6, 2024
23d383a
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 7, 2024
49ee4cb
address comments
zhenglongjiepheonix Jun 9, 2024
bc85aa9
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 9, 2024
247e789
[run-slow] mamba
zhenglongjiepheonix Jun 9, 2024
9e1fb0e
use_cache_position
zhenglongjiepheonix Jun 10, 2024
8a132ac
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 10, 2024
100b999
[run-slow] mamba
zhenglongjiepheonix Jun 10, 2024
2dc8986
[run-slow] mamba
zhenglongjiepheonix Jun 10, 2024
9732a62
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 11, 2024
c2c8e5d
[run-slow] mamba
zhenglongjiepheonix Jun 11, 2024
9120fbf
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 12, 2024
63568df
[run-slow] mamba
zhenglongjiepheonix Jun 12, 2024
0736da8
fix
zhenglongjiepheonix Jun 17, 2024
3141d26
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 17, 2024
584528e
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 20, 2024
f136d3c
cache in generate
zhenglongjiepheonix Jun 20, 2024
ec4a7a3
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jun 20, 2024
400feec
[run-slow] mamba
zhenglongjiepheonix Jun 20, 2024
de9182d
address comments
zhenglongjiepheonix Jun 27, 2024
38441cd
resolve conflict
zhenglongjiepheonix Jun 27, 2024
a254a09
[run-slow] mamba
zhenglongjiepheonix Jun 27, 2024
ac456ed
[run-slow] mamba
zhenglongjiepheonix Jun 27, 2024
3e95813
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jul 1, 2024
801e8d1
address comments
zhenglongjiepheonix Jul 1, 2024
b8af2a3
[run-slow] mamba
zhenglongjiepheonix Jul 1, 2024
b2e8a0b
fix
zhenglongjiepheonix Jul 8, 2024
708d302
fix
zhenglongjiepheonix Jul 8, 2024
97b1add
[run-slow] mamba
zhenglongjiepheonix Jul 8, 2024
2feeeb0
fix
zhenglongjiepheonix Jul 8, 2024
b045c38
fix conflict
zhenglongjiepheonix Jul 13, 2024
fcdc98f
[run-slow] mamba
zhenglongjiepheonix Jul 13, 2024
82b0c9b
fix cache name
zhenglongjiepheonix Jul 16, 2024
611e0d7
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jul 16, 2024
bc1563e
Merge remote-tracking branch 'upstream/main' into longjie/add_torch_c…
zhenglongjiepheonix Jul 16, 2024
78c8c1c
[run-slow] mamba
zhenglongjiepheonix Jul 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 54 additions & 23 deletions src/transformers/models/mamba/modeling_mamba.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import math
from dataclasses import dataclass
from typing import Any, Dict, Optional, Tuple, Union
from typing import Any, Dict, Optional, Tuple, Union, List

import torch
import torch.utils.checkpoint
Expand Down Expand Up @@ -68,8 +68,9 @@ class MambaCache:
Attributes:
seqlen_offset: int
dtype: torch.dtype
conv_states: Dict[int, torch.Tensor] # layer_idx -> [batch_size, intermediate_size, conv_kernel_size]
ssm_states: Dict[int, torch.Tensor] # layer_idx -> [batch_size, intermediate_size, ssm_state_size]
conv_state_cache: List[torch.Tensor] # layer_idx -> [batch_size, intermediate_size, conv_kernel_size]
ssm_state_cache: List[torch.Tensor] # layer_idx -> [batch_size, intermediate_size, ssm_state_size]
is_cache_initialized: List[bool]
"""

def __init__(
Expand All @@ -81,14 +82,47 @@ def __init__(
ssm_state_size = config.state_size
conv_kernel_size = config.conv_kernel

self.conv_states = {
i: torch.zeros(batch_size, intermediate_size, conv_kernel_size, device=device, dtype=dtype)
for i in range(config.num_hidden_layers)
}
self.ssm_states = {
i: torch.zeros(batch_size, intermediate_size, ssm_state_size, device=device, dtype=dtype)
for i in range(config.num_hidden_layers)
}
self.conv_state_cache : List[torch.Tensor] = []
self.ssm_state_cache : List[torch.Tensor] = []

self.is_cache_initialized = [False for _ in range(config.num_hidden_layers)]
Comment thread
zhenglongjiepheonix marked this conversation as resolved.
Resolved

for i in range(config.num_hidden_layers):
new_layer_conv_cache = torch.zeros(batch_size, intermediate_size, conv_kernel_size, device=device, dtype=dtype)
new_layer_ssm_cache = torch.zeros(batch_size, intermediate_size, ssm_state_size, device=device, dtype=dtype)
torch._dynamo.mark_static_address(new_layer_conv_cache)
torch._dynamo.mark_static_address(new_layer_ssm_cache)
self.conv_state_cache.append(new_layer_conv_cache)
self.ssm_state_cache.append(new_layer_ssm_cache)

def update_conv_state(self, layer_idx : int, hidden_states : torch.Tensor) -> torch.Tensor:
seq_len = hidden_states.shape[-1]
if seq_len > 1:
self.conv_state_cache[layer_idx] += hidden_states
return self.conv_state_cache[layer_idx]

# decoding phase
conv_state = self.conv_state_cache[layer_idx]
new_conv_state = conv_state.roll(shifts=-1, dims=-1)
new_conv_state[:, :, -1] = hidden_states[:, :, 0]
self.conv_state_cache[layer_idx].zero_()
self.conv_state_cache[layer_idx] += new_conv_state
return self.conv_state_cache[layer_idx]

def get_conv_sate(self, layer_idx):
return self.conv_state_cache[layer_idx]

def update_ssm_state(self, layer_idx : int, new_ssm_state : torch.Tensor):
self.ssm_state_cache[layer_idx].zero_()
self.ssm_state_cache[layer_idx] += new_ssm_state
self.is_cache_initialized[layer_idx] = True
return self.ssm_state_cache[layer_idx]

def get_ssm_sate(self, layer_idx):
return self.ssm_state_cache[layer_idx]

def is_initialized(self, layer_idx):
return self.is_cache_initialized[layer_idx]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this can be checked with cache_postiions instead no?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is fine just like we need a flag in whisper, here cache_positions is not so meaningful because we always know how to update and get the cache even if cache_positions is not passed

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Theoretically yes, but this is adding some complexity, which is not needed in the cache API. Checking the cache positions is more reliable, and is what we want to go with.

  • you don't have to reset and set another tensor
    which is also a win

Let's just use the cache positions



class MambaMixer(nn.Module):
Expand Down Expand Up @@ -243,13 +277,12 @@ def slow_forward(self, input_states, cache_params: Optional[MambaCache]=None):

# 2. Convolution sequence transformation
if cache_params is not None:
ssm_state = cache_params.ssm_states[self.layer_idx].clone()
# ssm_state = cache_params.ssm_states[self.layer_idx].clone()
ssm_state = cache_params.get_ssm_sate(self.layer_idx).clone()
ssm_state = ssm_state.to(hidden_states.device)
if cache_params.seqlen_offset > 0:
conv_state = cache_params.conv_states[self.layer_idx] # [batch, intermediate_size, conv_kernel_size]
conv_state = torch.roll(conv_state, shifts=-1, dims=-1)
conv_state[:, :, -1] = hidden_states[:, :, 0]
cache_params.conv_states[self.layer_idx].copy_(conv_state)
Comment thread
zhenglongjiepheonix marked this conversation as resolved.
Resolved

if cache_params.is_initialized(self.layer_idx):
conv_state = cache_params.update_conv_state(self.layer_idx, hidden_states)
hidden_states = torch.sum(conv_state * self.conv1d.weight[:, 0, :], dim=-1)
if self.use_conv_bias:
hidden_states += self.conv1d.bias
Expand All @@ -259,7 +292,7 @@ def slow_forward(self, input_states, cache_params: Optional[MambaCache]=None):
hidden_states,
(self.conv_kernel_size - hidden_states.shape[-1], 0)
)
cache_params.conv_states[self.layer_idx].copy_(conv_state)
cache_params.update_conv_state(self.layer_idx, conv_state)
hidden_states = self.act(self.conv1d(hidden_states)[..., :seq_len]) # [batch, intermediate_size, seq_len]
else:
ssm_state = torch.zeros(
Expand Down Expand Up @@ -294,7 +327,7 @@ def slow_forward(self, input_states, cache_params: Optional[MambaCache]=None):
scan_output = (scan_output * self.act(gate))

if cache_params is not None:
cache_params.ssm_states[self.layer_idx].copy_(ssm_state)
cache_params.update_ssm_state(self.layer_idx, ssm_state)

# 4. Final linear projection
contextualized_states = self.out_proj(scan_output.transpose(1, 2)) # [batch, seq_len, hidden_size]
Expand Down Expand Up @@ -579,8 +612,6 @@ def forward(
if output_hidden_states:
all_hidden_states = all_hidden_states + (hidden_states,)

if use_cache:
cache_params.seqlen_offset += inputs_embeds.shape[1]

hidden_states = self.norm_f(hidden_states)

Expand Down Expand Up @@ -633,10 +664,10 @@ def _update_model_kwargs_for_generation(
return model_kwargs

def prepare_inputs_for_generation(
self, input_ids, cache_params: Optional[MambaCache] = None, inputs_embeds=None, attention_mask=None, **kwargs
self, input_ids, cache_params: Optional[MambaCache] = None, inputs_embeds=None, **kwargs
):
# only last token for inputs_ids if the state is passed along.
if cache_params is not None:
if cache_params is not None and cache_params.is_initialized(0):
input_ids = input_ids[:, -1].unsqueeze(-1)

if inputs_embeds is not None and cache_params is None:
Expand Down