From a9703d0c4d1032bc8df3150fa3463b108fe90ec2 Mon Sep 17 00:00:00 2001 From: Ivar Flakstad <69173633+ivarflakstad@users.noreply.github.com> Date: Sun, 29 Jun 2025 20:50:41 +0200 Subject: [PATCH 1/2] These MoEs do not support static cache (atm) --- src/transformers/models/deepseek_v3/modeling_deepseek_v3.py | 5 ++++- src/transformers/models/deepseek_v3/modular_deepseek_v3.py | 6 ++++++ src/transformers/models/dots1/modeling_dots1.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index 541ae6669e92..c052c6c26e4a 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -510,7 +510,7 @@ class DeepseekV3PreTrainedModel(PreTrainedModel): _supports_flex_attn = True _supports_cache_class = True _supports_quantized_cache = True - _supports_static_cache = True + _supports_static_cache = False _supports_attention_backend = True def _init_weights(self, module): @@ -531,6 +531,8 @@ def _init_weights(self, module): @auto_docstring class DeepseekV3Model(DeepseekV3PreTrainedModel): + _supports_static_cache = False + _keys_to_ignore_on_load_unexpected = [r"model\.layers\.61.*"] def __init__(self, config: DeepseekV3Config): @@ -664,6 +666,7 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + _supports_static_cache = False def __init__(self, config): super().__init__(config) diff --git a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py index 73f3acc2047e..3fababd914c5 100644 --- a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py @@ -338,6 +338,8 @@ def __init__(self, config: DeepseekV3Config, layer_idx: int): class DeepseekV3PreTrainedModel(LlamaPreTrainedModel): + _supports_static_cache = False + def _init_weights(self, module): std = self.config.initializer_range if isinstance(module, nn.Linear): @@ -355,10 +357,14 @@ def _init_weights(self, module): class DeepseekV3Model(LlamaModel): + _supports_static_cache = False + _keys_to_ignore_on_load_unexpected = [r"model\.layers\.61.*"] class DeepseekV3ForCausalLM(LlamaForCausalLM): + _supports_static_cache = False + pass diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index 58b805cca613..c0c730450fbc 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -430,7 +430,7 @@ class Dots1PreTrainedModel(PreTrainedModel): _supports_flex_attn = True _supports_cache_class = True _supports_quantized_cache = True - _supports_static_cache = True + _supports_static_cache = False _supports_attention_backend = True def _init_weights(self, module): From ccedf5be4bb5cf8fca98765e795a421fec4af96d Mon Sep 17 00:00:00 2001 From: Ivar Flakstad <69173633+ivarflakstad@users.noreply.github.com> Date: Mon, 28 Jul 2025 18:04:49 +0200 Subject: [PATCH 2/2] Remove setting that should be inherited. Formatting. --- src/transformers/models/deepseek_v3/modeling_deepseek_v3.py | 3 --- src/transformers/models/deepseek_v3/modular_deepseek_v3.py | 4 ---- src/transformers/models/dots1/modeling_dots1.py | 1 - 3 files changed, 8 deletions(-) diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index b08336473412..d6a27b41c15d 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -512,8 +512,6 @@ def _init_weights(self, module): @auto_docstring class DeepseekV3Model(DeepseekV3PreTrainedModel): - _can_compile_fullgraph = False - _keys_to_ignore_on_load_unexpected = [r"model\.layers\.61.*"] def __init__(self, config: DeepseekV3Config): @@ -598,7 +596,6 @@ class DeepseekV3ForCausalLM(DeepseekV3PreTrainedModel, GenerationMixin): _tied_weights_keys = ["lm_head.weight"] _tp_plan = {"lm_head": "colwise_rep"} _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} - _can_compile_fullgraph = False def __init__(self, config): super().__init__(config) diff --git a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py index d136ca1b88e3..ef6faf4f2292 100644 --- a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py @@ -347,14 +347,10 @@ def _init_weights(self, module): class DeepseekV3Model(LlamaModel): - _can_compile_fullgraph = False - _keys_to_ignore_on_load_unexpected = [r"model\.layers\.61.*"] class DeepseekV3ForCausalLM(LlamaForCausalLM): - _can_compile_fullgraph = False - pass diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index 1b9632e5b76a..476f8b1bffc4 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -417,7 +417,6 @@ class Dots1PreTrainedModel(PreTrainedModel): _supports_flash_attn = True _supports_sdpa = True _supports_flex_attn = True - _can_compile_fullgraph = False _supports_attention_backend = True _can_record_outputs = {