Skip to content
Merged
Show file tree
Hide file tree
Changes from 43 commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
57b5870
fully remove from the mask api
Cyrilvallez Feb 20, 2026
b9672ff
fully remove from cache api
Cyrilvallez Feb 20, 2026
88a0b2e
remove it from llama + modulars
Cyrilvallez Feb 20, 2026
f5dc06c
remove them from all modulars of llama descendants
Cyrilvallez Feb 20, 2026
b792266
reapply all those modualrs
Cyrilvallez Feb 20, 2026
c2fde0a
a few more
Cyrilvallez Feb 20, 2026
0f12a85
style
Cyrilvallez Feb 23, 2026
3a33d19
merge main
Cyrilvallez Feb 24, 2026
b2d91fc
style
Cyrilvallez Feb 24, 2026
49b47fe
better get_seq_length
Cyrilvallez Feb 24, 2026
2a21bfa
merge main again
Cyrilvallez Feb 24, 2026
94a44d0
fix signatures
Cyrilvallez Feb 24, 2026
417da7a
fix device
Cyrilvallez Feb 24, 2026
beffc18
win against dynamo
Cyrilvallez Feb 24, 2026
924fb63
merge main AGAIN
Cyrilvallez Feb 24, 2026
f15531f
compatible with cuda graphs
Cyrilvallez Feb 24, 2026
3102c0b
add args to absorb
Cyrilvallez Feb 24, 2026
5ce0c8a
fix lfm2
Cyrilvallez Feb 24, 2026
318f621
fix executorch
Cyrilvallez Feb 24, 2026
0cf77d6
fix zamba2
Cyrilvallez Feb 24, 2026
e86122e
fix encoder-decoder
Cyrilvallez Feb 24, 2026
4b21a57
fix
Cyrilvallez Feb 24, 2026
367f44d
fix t5gemma
Cyrilvallez Feb 25, 2026
8d9a5d4
oupsi
Cyrilvallez Feb 25, 2026
82bb20c
oupsi again
Cyrilvallez Feb 25, 2026
c066e27
paddleocr
Cyrilvallez Feb 25, 2026
32fd1ee
fix lfms
Cyrilvallez Feb 25, 2026
49cf111
fix export
Cyrilvallez Feb 25, 2026
3605140
more export fix
Cyrilvallez Feb 25, 2026
ae59554
lfm2
Cyrilvallez Feb 25, 2026
418a02d
fix cuda graph super weird issue
Cyrilvallez Feb 26, 2026
ece2c78
oupsi forgot to add it elsewhere
Cyrilvallez Feb 26, 2026
ca733ab
fix
Cyrilvallez Feb 26, 2026
4712b29
fix all finally
Cyrilvallez Feb 26, 2026
29421b0
style
Cyrilvallez Feb 26, 2026
0e2b882
fix break
Cyrilvallez Feb 26, 2026
883fd05
Merge branch 'main' into remove-cache-pos
Cyrilvallez Feb 27, 2026
8e79feb
latest model
Cyrilvallez Feb 27, 2026
0d6f071
style
Cyrilvallez Feb 27, 2026
c2d8180
add doc
Cyrilvallez Feb 27, 2026
5385b2f
Merge branch 'main' into remove-cache-pos
Cyrilvallez Feb 27, 2026
428cce2
Merge branch 'main' into remove-cache-pos
Cyrilvallez Mar 2, 2026
bec6545
fix audioflamingo
Cyrilvallez Mar 2, 2026
3b3f519
doc
Cyrilvallez Mar 2, 2026
4056eb4
bc on internal mask functions
Cyrilvallez Mar 2, 2026
8f6ca3d
oupsi version
Cyrilvallez Mar 2, 2026
b3c0b33
Merge branch 'main' into remove-cache-pos
Cyrilvallez Mar 3, 2026
5eb5459
Merge branch 'main' into remove-cache-pos
Cyrilvallez Mar 4, 2026
7ac53a9
nemotron
Cyrilvallez Mar 4, 2026
598a83c
Merge branch 'main' into remove-cache-pos
Cyrilvallez Mar 4, 2026
9030f13
eurobert
Cyrilvallez Mar 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 65 additions & 83 deletions src/transformers/cache_utils.py

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions src/transformers/generation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -563,8 +563,8 @@ def prepare_inputs_for_generation(
causal_mask_creation_function = getattr(self, "create_masks_for_generate", create_masks_for_generate)
attention_mask = causal_mask_creation_function(
config=self.config,
# we only need batch size, seq_length and dtype here - we don't care about the values of the embeddings
inputs_embeds=torch.empty((batch_size, sequence_length), dtype=self.dtype),
# we only need batch size, seq_length, dtype and device here - so we pass a 0-sized tensor with only the metadata
inputs_embeds=torch.empty((batch_size, sequence_length, 0), dtype=self.dtype, device=input_ids.device),
attention_mask=attention_mask,
cache_position=model_inputs.get("cache_position"),
past_key_values=model_inputs.get("past_key_values"),
Expand Down
60 changes: 50 additions & 10 deletions src/transformers/integrations/executorch.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
DynamicSlidingWindowLayer,
EncoderDecoderCache,
StaticCache,
StaticLayer,
StaticSlidingWindowLayer,
)
from ..generation.configuration_utils import GenerationConfig
from ..modeling_utils import PreTrainedModel
Expand Down Expand Up @@ -521,14 +523,22 @@ def __init__(
# Initialize the static cache
self.model = model
self.static_cache = StaticCache(max_cache_len=max_cache_len, config=config)
# Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by
# simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported
for i, layer in enumerate(self.static_cache.layers):
if isinstance(layer, StaticSlidingWindowLayer):
self.static_cache.layers[i] = StaticLayer(layer.max_cache_len)
Comment on lines +526 to +530

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we add something in the docs to clarify, this seems like something unseemingly hard to catch by outsiders - and I don't think we will remove this limitation any time soon 😓

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could, but note that this is ALREADY the case on main, and it looks like nobody noticed/raised issue... It would generate garbage beyond the sliding window, it's just explicit in the code now!
To solve it, we could either have a mimimal version of sliding cache for export, which would work ONLY for decoding (i.e. 1 new tokens all the time, without being able to feed more than 1 token after prefill), or we could juste use full cache with proper sliding masking, but we loose some compute

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea, still think it would be nice to clarify somewhere. True, it's also broken in main, maybe out of scope for this PR

head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
num_heads = getattr(config, "num_key_value_heads", config.num_attention_heads)
dtype = self.model.dtype
# We need this call to initialize all the layers (otherwise it's done lazily, which is not exportable)
self.static_cache.early_initialization(batch_size, num_heads, head_dim, dtype, device)
for i in range(len(self.static_cache)):
self.register_buffer(f"key_cache_{i}", self.static_cache.layers[i].keys, persistent=False)
self.register_buffer(f"value_cache_{i}", self.static_cache.layers[i].values, persistent=False)

# Register cache buffers to make them exportable
for i, layer in enumerate(self.static_cache.layers):
self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False)
self.register_buffer(f"value_cache_{i}", layer.values, persistent=False)
self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False)

def forward(
self,
Expand Down Expand Up @@ -557,6 +567,12 @@ def forward(
The adapter matches the model's forward signature with that in `executorch/extension/llm/runner`,
ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box.
"""
# Start by resetting static cache (it's needed to be able to run several generations with the same exported program,
# as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was
# already exported)
for layer in self.static_cache.layers:
layer.cumulative_length.copy_(cache_position[0:1])

past_key_values = self.static_cache

outs = self.model(
Expand Down Expand Up @@ -687,16 +703,22 @@ def __init__(

# Initialize the cache
self.cache = StaticCache(config=config, max_cache_len=max_cache_len)
# Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by
# simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported
for i, layer in enumerate(self.cache.layers):
if isinstance(layer, StaticSlidingWindowLayer):
self.cache.layers[i] = StaticLayer(layer.max_cache_len)
head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
num_heads = getattr(config, "num_key_value_heads", config.num_attention_heads)
dtype = self.model.dtype
# We need this call to initialize all the layers (otherwise it's done lazily, which is not exportable)
self.cache.early_initialization(batch_size, num_heads, head_dim, dtype, device)

# Register all key and value cache tensors as buffers
for i in range(len(self.cache)):
self.register_buffer(f"key_cache_{i}", self.cache.layers[i].keys, persistent=False)
self.register_buffer(f"value_cache_{i}", self.cache.layers[i].values, persistent=False)
# Register cache buffers to make them exportable
for i, layer in enumerate(self.cache.layers):
self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False)
self.register_buffer(f"value_cache_{i}", layer.values, persistent=False)
self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False)

def forward(
self,
Expand All @@ -715,6 +737,12 @@ def forward(
Returns:
torch.Tensor: Logits output from the model.
"""
# Start by resetting static cache (it's needed to be able to run several generations with the same exported program,
# as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was
# already exported)
for layer in self.cache.layers:
layer.cumulative_length.copy_(cache_position[0:1])

# Forward pass with the model
outputs = self.model(
input_ids=input_ids,
Expand Down Expand Up @@ -830,6 +858,11 @@ def __init__(self, model, max_static_cache_length, batch_size):

# Initialize static cache for decoder and DynamicCache for encoder
self.static_cache = StaticCache(config=self.config, max_cache_len=max_static_cache_length)
# Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by
# simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported
for i, layer in enumerate(self.static_cache.layers):
if isinstance(layer, StaticSlidingWindowLayer):
self.static_cache.layers[i] = StaticLayer(layer.max_cache_len)
head_dim = getattr(self.config, "head_dim", self.config.hidden_size // self.config.num_attention_heads)
num_heads = getattr(self.config, "num_key_value_heads", self.config.num_attention_heads)
self.static_cache.early_initialization(batch_size, num_heads, head_dim, torch.float32, model_device)
Expand All @@ -838,11 +871,18 @@ def __init__(self, model, max_static_cache_length, batch_size):
register_dynamic_cache_export_support()

# Register cache buffers to make them exportable
for i in range(len(self.static_cache)):
self.register_buffer(f"key_cache_{i}", self.static_cache.layers[i].keys, persistent=False)
self.register_buffer(f"value_cache_{i}", self.static_cache.layers[i].values, persistent=False)
for i, layer in enumerate(self.static_cache.layers):
self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False)
self.register_buffer(f"value_cache_{i}", layer.values, persistent=False)
self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False)

def forward(self, decoder_input_ids, encoder_hidden_states, cache_position):
# Start by resetting static cache (it's needed to be able to run several generations with the same exported program,
# as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was
# already exported)
for layer in self.static_cache.layers:
layer.cumulative_length.copy_(cache_position[0:1])

# Get outputs from decoder
outputs = self.decoder(
input_ids=decoder_input_ids,
Expand Down
Loading