From 57b58709be73f45c419098b46ad0ef4f67c854a9 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 17:15:09 +0100 Subject: [PATCH 01/42] fully remove from the mask api --- src/transformers/cache_utils.py | 25 ++--- src/transformers/masking_utils.py | 174 +++++++++++++++++------------- 2 files changed, 108 insertions(+), 91 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 530ceda9d593..7aee3e718fa8 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -46,7 +46,7 @@ def update( ) -> tuple[torch.Tensor, torch.Tensor]: ... @abstractmethod - def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: ... + def get_mask_sizes(self, query_length: int) -> tuple[int, int]: ... @abstractmethod def get_seq_length(self) -> int: ... @@ -121,10 +121,9 @@ def update( self.values = torch.cat([self.values, value_states], dim=-2) return self.keys, self.values - def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length() + query_length return kv_length, kv_offset @@ -216,9 +215,8 @@ def update( # Return the full states return full_key_states, full_value_states - def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the attention mask""" - query_length = cache_position.shape[0] is_full = self.cumulative_length >= self.sliding_window kv_offset = max(self.cumulative_length - self.sliding_window + 1, 0) @@ -345,7 +343,7 @@ def update( self.values[:, :, cache_position] = value_states return self.keys, self.values - def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the attention mask""" kv_offset = 0 kv_length = self.max_cache_len @@ -463,9 +461,8 @@ def update( # we should return the whole states instead of `self.keys/values` here, as otherwise we lose some context return full_key_states, full_value_states - def get_mask_sizes(self, cache_position: torch.Tensor) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the attention mask""" - query_length = cache_position.shape[0] sliding_window = self.max_cache_len is_full = self.cumulative_length >= self.max_cache_len @@ -817,17 +814,17 @@ def get_seq_length(self, layer_idx: int = 0) -> int: return 0 return self.layers[layer_idx].get_seq_length() - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ # For DynamicCache, where the layers are created at runtime -> if it was not yet created, the size is - # simply the shape of `cache_position` + # simply the query_length if layer_idx >= len(self.layers): - return cache_position.shape[0], 0 - return self.layers[layer_idx].get_mask_sizes(cache_position) + return query_length, 0 + return self.layers[layer_idx].get_mask_sizes(query_length) def get_max_cache_shape(self, layer_idx: int = 0) -> int: """Returns maximum sequence length of the cache object. Dynamic caches do not have a maximum length.""" @@ -1278,8 +1275,8 @@ def get_max_cache_shape(self) -> int: """Returns the maximum sequence length (i.e. max capacity) of the cache object""" return self.self_attention_cache.get_max_cache_shape() - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: - return self.self_attention_cache.get_mask_sizes(cache_position, layer_idx) + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: + return self.self_attention_cache.get_mask_sizes(query_length, layer_idx) @property def is_sliding(self): diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index ee594262e51f..4370bd6faf4d 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -366,8 +366,9 @@ def _non_vmap_expansion_sdpa( def sdpa_mask( batch_size: int, - cache_position: torch.Tensor, + q_length: int, kv_length: int, + q_offset: int = 0, kv_offset: int = 0, mask_function: Callable = causal_mask_function, attention_mask: torch.Tensor | None = None, @@ -376,6 +377,7 @@ def sdpa_mask( allow_is_bidirectional_skip: bool = False, allow_torch_fix: bool = True, use_vmap: bool = False, + device: torch.device | str = "cpu", **kwargs, ) -> torch.Tensor | None: """ @@ -386,12 +388,14 @@ def sdpa_mask( Args: batch_size (`int`): The batch size of the input sequence. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + q_length (`int`): + The size that the query states will have during the attention computation. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. + q_offset (`int`, optional): + An optional offset to indicate at which first position the query states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): @@ -411,6 +415,8 @@ def sdpa_mask( use_vmap (`bool`, optional): Whether to use `vmap` during the mask construction or not. Allows powerful custom patterns that may not be index-based (for the cost of speed performance). By default `False`. + device (`torch.device` or `str`, optional): + An optional device to create the mask on. ## Creating a simple causal mask: @@ -426,7 +432,7 @@ def sdpa_mask( You can do ```python - >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5) + >>> sdpa_mask(batch_size=1, q_length=5, kv_length=5) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], @@ -447,7 +453,7 @@ def sdpa_mask( You can do ```python - >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5, mask_function=sliding_window_causal_mask_function(3)) + >>> sdpa_mask(batch_size=1, q_length=5, kv_length=5, mask_function=sliding_window_causal_mask_function(3)) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], @@ -468,7 +474,7 @@ def sdpa_mask( You can do ```python - >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5, mask_function=chunked_causal_mask_function(3, torch.zeros(1, dtype=int))) + >>> sdpa_mask(batch_size=1, q_length=5, kv_length=5, mask_function=chunked_causal_mask_function(3, torch.zeros(1, dtype=int))) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], @@ -477,8 +483,6 @@ def sdpa_mask( ``` """ - q_length = cache_position.shape[0] - # Potentially pad the 2D mask padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset) @@ -494,17 +498,16 @@ def sdpa_mask( if padding_mask is not None: mask_function = and_masks(mask_function, padding_mask_function(padding_mask)) - batch_arange = torch.arange(batch_size, device=cache_position.device) - head_arange = torch.arange(1, device=cache_position.device) - # Similar to `kv_arange = torch.arange(start=kv_offset, end=kv_offset + kv_length, device=cache_position.device)` - # but without data-dependent slicing (i.e. torch.compile friendly) - kv_arange = torch.arange(kv_length, device=cache_position.device) + kv_offset + batch_arange = torch.arange(batch_size, device=device) + head_arange = torch.arange(1, device=device) + q_arange = torch.arange(q_length, device=device) + q_offset + kv_arange = torch.arange(kv_length, device=device) + kv_offset # Actual mask creation # Option 1: Fast non-vmap mask creation (default) if not use_vmap: # Apply mask function element-wise through broadcasting - attention_mask = mask_function(*_non_vmap_expansion_sdpa(batch_arange, head_arange, cache_position, kv_arange)) + attention_mask = mask_function(*_non_vmap_expansion_sdpa(batch_arange, head_arange, q_arange, kv_arange)) # Expand the mask to match batch size and query length if they weren't used in the mask function attention_mask = attention_mask.expand(batch_size, -1, q_length, kv_length) @@ -514,7 +517,7 @@ def sdpa_mask( # scalar tensor (it internally calls `.item()` which vmap does not allow, but this context works around it # We don't need to add an offset to the mask_function either, as we vmap directly the correct indices for k and kv indices with TransformGetItemToIndex(): - attention_mask = _vmap_expansion_sdpa(mask_function)(batch_arange, head_arange, cache_position, kv_arange) + attention_mask = _vmap_expansion_sdpa(mask_function)(batch_arange, head_arange, q_arange, kv_arange) # Option 3: Error out since it indicates that the user did something custom, which they shouldn't have (torch<2.6) else: @@ -533,14 +536,16 @@ def sdpa_mask( def eager_mask( batch_size: int, - cache_position: torch.Tensor, + q_length: int, kv_length: int, + q_offset: int = 0, kv_offset: int = 0, mask_function: Callable = causal_mask_function, attention_mask: torch.Tensor | None = None, dtype: torch.dtype = torch.float32, allow_is_bidirectional_skip: bool = False, use_vmap: bool = False, + device: torch.device | str = "cpu", **kwargs, ) -> torch.Tensor: """ @@ -551,10 +556,12 @@ def eager_mask( Args: batch_size (`int`): The batch size of the input sequence. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + q_length (`int`): + The size that the query states will have during the attention computation. kv_length (`int`): The size that the key and value states will have during the attention computation. + q_offset (`int`, optional): + An optional offset to indicate at which first position the query states will refer to. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): @@ -569,14 +576,17 @@ def eager_mask( use_vmap (`bool`, optional): Whether to use `vmap` during the mask construction or not. Allows powerful custom patterns that may not be index-based (for the cost of speed performance). By default `False`. + device (`torch.device` or `str`, optional): + An optional device to create the mask on. """ # The masks for eager attention are simply boolean mask from sdpa, casted to 0 and -inf _ = kwargs.pop("allow_is_causal_skip", None) _ = kwargs.pop("allow_torch_fix", None) mask = sdpa_mask( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_function, attention_mask=attention_mask, @@ -584,6 +594,7 @@ def eager_mask( allow_is_bidirectional_skip=allow_is_bidirectional_skip, allow_torch_fix=False, use_vmap=use_vmap, + device=device, **kwargs, ) # only bidirectional masks can be skipped, otherwise we convert bool -> float @@ -596,8 +607,9 @@ def eager_mask( def flash_attention_mask( batch_size: int, - cache_position: torch.Tensor, + q_length: int, kv_length: int, + q_offset: int = 0, kv_offset: int = 0, mask_function: Callable = causal_mask_function, attention_mask: torch.Tensor | None = None, @@ -611,10 +623,12 @@ def flash_attention_mask( Args: batch_size (`int`): The batch size of the input sequence. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + q_length (`int`): + The size that the query states will have during the attention computation. kv_length (`int`): The size that the key and value states will have during the attention computation. + q_offset (`int`, optional): + An optional offset to indicate at which first position the query states will refer to. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): @@ -635,11 +649,13 @@ def flash_attention_mask( def flex_attention_mask( batch_size: int, - cache_position: torch.Tensor, + q_length: int, kv_length: int, + q_offset: int = 0, kv_offset: int = 0, mask_function: Callable = causal_mask_function, attention_mask: torch.Tensor | None = None, + device: torch.device | str = "cpu", **kwargs, ) -> BlockMask: """ @@ -649,19 +665,21 @@ def flex_attention_mask( Args: batch_size (`int`): The batch size of the input sequence. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + q_length (`int`): + The size that the query states will have during the attention computation. kv_length (`int`): The size that the key and value states will have during the attention computation. + q_offset (`int`, optional): + An optional offset to indicate at which first position the query states will refer to. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length) + device (`torch.device` or `str`, optional): + An optional device to create the mask on. """ - q_length, q_offset = cache_position.shape[0], cache_position[0] - # Potentially add the padding 2D mask if attention_mask is not None: # Older torch (2.5.x) cannot handle sequences not in multiples of 128 (default block size) @@ -684,7 +702,7 @@ def flex_attention_mask( H=None, Q_LEN=q_length, KV_LEN=kv_length, - device=cache_position.device, + device=device, _compile=_is_torch_greater_or_equal_than_2_6, ) return block_mask @@ -743,7 +761,6 @@ def _preprocess_mask_arguments( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | BlockMask | None, - cache_position: torch.Tensor, past_key_values: Cache | None, position_ids: torch.Tensor | None, layer_idx: int | None, @@ -761,8 +778,6 @@ def _preprocess_mask_arguments( attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) @@ -779,14 +794,18 @@ def _preprocess_mask_arguments( packed_sequence_mask (`torch.Tensor`, optional): In case we detected packed sequence format, this is a tensor where each similar integer indicates that the tokens belong to the same sequence. + q_length (`int`): + The size that the query states will have during the attention computation. kv_length (`int`): The size that the key and value states will have during the attention computation. + q_offset (`int`, optional): + An optional offset to indicate at which first position the query states will refer to. kv_offset (`int`): An offset to indicate at which first position the key and values states will refer to. """ # If the mask is already 4D, simply return as-is (it was already prepared, or it is custom) if isinstance(attention_mask, (torch.Tensor, BlockMask)) and len(attention_mask.shape) == 4: - return True, attention_mask, None, None, None + return True, attention_mask, None, None, None, None, None # For TGI/vLLM backends, or other custom attention without equivalent mask creation: we don't need a mask! # Note: it's not ideal to check the `_global_mapping` attribute instead of the object itself, however otherwise @@ -794,20 +813,23 @@ def _preprocess_mask_arguments( # with `torch._dynamo.exc.Unsupported: 'inline in skipfiles:Mapping.__contains__ | __contains__, skipped # according trace_rules.lookup SKIP_DIRS'` -- can be removed when we require Python>=3.11 if config._attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping: - return True, None, None, None, None + return True, None, None, None, None, None, None # Move the mask to correct device, and potentially switch dtype for efficiency if attention_mask is not None and attention_mask.ndim == 2: - attention_mask = attention_mask.to(device=cache_position.device, dtype=torch.bool) + attention_mask = attention_mask.to(device=inputs_embeds.device, dtype=torch.bool) + q_length = inputs_embeds.shape[1] # If using a cache, it can give all information about mask sizes based on seen tokens if past_key_values is not None: - kv_length, kv_offset = past_key_values.get_mask_sizes(cache_position, layer_idx) + q_offset = past_key_values.get_seq_length() + kv_length, kv_offset = past_key_values.get_mask_sizes(q_length, layer_idx) # Otherwise, we infer based on our input else: + q_offset = 0 # 1. Rely on input directly if attention_mask is None: - kv_length, kv_offset = inputs_embeds.shape[1], 0 + kv_length, kv_offset = q_length, 0 # 2. Rely on the mask instead - needed for special cases like prefix tuning in PEFT # # This is a very unique and special case where an encoder utilizes a cache and expects its length @@ -827,7 +849,7 @@ def _preprocess_mask_arguments( position_ids = position_ids.expand(batch_size, -1) packed_sequence_mask = find_packed_sequence_indices(position_ids) - return False, attention_mask, packed_sequence_mask, kv_length, kv_offset + return False, attention_mask, packed_sequence_mask, q_length, kv_length, q_offset, kv_offset @deprecate_kwarg("input_embeds", version="5.6.0", new_name="inputs_embeds") @@ -835,7 +857,7 @@ def create_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, + cache_position: torch.Tensor, # not used anymore but kept for BC past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -856,7 +878,7 @@ def create_causal_mask( The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + Deprecated and unused. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) @@ -885,13 +907,13 @@ def create_causal_mask( else: layer_idx = 0 - early_exit, attention_mask, packed_sequence_mask, kv_length, kv_offset = _preprocess_mask_arguments( - config, inputs_embeds, attention_mask, cache_position, past_key_values, position_ids, layer_idx + early_exit, attention_mask, packed_sequence_mask, q_length, kv_length, q_offset, kv_offset = ( + _preprocess_mask_arguments(config, inputs_embeds, attention_mask, past_key_values, position_ids, layer_idx) ) if early_exit: return attention_mask - batch_size, dtype = inputs_embeds.shape[0], inputs_embeds.dtype + batch_size, dtype, device = inputs_embeds.shape[0], inputs_embeds.dtype, inputs_embeds.device mask_factory_function = causal_mask_function mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] @@ -904,7 +926,7 @@ def create_causal_mask( # TODO: cyril -> probably revisit and remove this, but a lot of tests rely on it if _is_torch_xpu_available: # Do not allow skip if we are compiling for decoding, but for prefill, we still allow skip to optimization the perf of 1st token generation - allow_is_causal_skip = not (getattr(past_key_values, "is_compileable", False) and cache_position.shape[0] == 1) + allow_is_causal_skip = not (getattr(past_key_values, "is_compileable", False) and q_length == 1) else: allow_is_causal_skip = not getattr(past_key_values, "is_compileable", False) @@ -932,8 +954,9 @@ def create_causal_mask( # We now create the mask causal_mask = mask_interface( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_factory_function, attention_mask=attention_mask, @@ -941,6 +964,7 @@ def create_causal_mask( dtype=dtype, # Additional kwarg for eager config=config, # Pass the config as well, in case someone wants to easily have their own mask_interface use_vmap=use_vmap, # Short-circuit to non-vmap expansions for the mask + device=device, ) return causal_mask @@ -977,19 +1001,15 @@ def create_bidirectional_mask( An optional mask function to combine with the base mask function (by doing the intersection of both). This is useful to easily overlay another mask on top, for example for image tokens handling. """ - # Due to the logic surrounding `cache_position` in inferring query-related information, we - # construct a dummy tensor imitating initial positions - cache_position = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device, dtype=torch.long) - embeds = encoder_hidden_states if encoder_hidden_states is not None else inputs_embeds # We ignore a few irrelevant arguments at the end as we do not have a (growing) cache here - early_exit, attention_mask, _, kv_length, kv_offset = _preprocess_mask_arguments( - config, embeds, attention_mask, cache_position, None, None, 0 + early_exit, attention_mask, _, q_length, kv_length, q_offset, kv_offset = _preprocess_mask_arguments( + config, embeds, attention_mask, None, None, 0 ) if early_exit: return attention_mask - batch_size, dtype = embeds.shape[0], embeds.dtype + batch_size, dtype, device = embeds.shape[0], embeds.dtype, embeds.device mask_factory_function = bidirectional_mask_function mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] @@ -1019,8 +1039,9 @@ def create_bidirectional_mask( # We now create the mask attention_mask = mask_interface( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_factory_function, attention_mask=attention_mask, @@ -1030,6 +1051,7 @@ def create_bidirectional_mask( dtype=dtype, # Additional kwarg for eager config=config, # Pass the config as well, in case someone wants to easily have their own mask_interface use_vmap=use_vmap, # Short-circuit to non-vmap expansions for the mask + device=device, ) return attention_mask @@ -1039,7 +1061,7 @@ def create_sliding_window_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, + cache_position: torch.Tensor, # not used anymore but kept for BC past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -1061,7 +1083,7 @@ def create_sliding_window_causal_mask( The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + Deprecated and unused. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) @@ -1090,8 +1112,8 @@ def create_sliding_window_causal_mask( else: layer_idx = 0 - early_exit, attention_mask, packed_sequence_mask, kv_length, kv_offset = _preprocess_mask_arguments( - config, inputs_embeds, attention_mask, cache_position, past_key_values, position_ids, layer_idx + early_exit, attention_mask, packed_sequence_mask, q_length, kv_length, q_offset, kv_offset = ( + _preprocess_mask_arguments(config, inputs_embeds, attention_mask, past_key_values, position_ids, layer_idx) ) if early_exit: return attention_mask @@ -1100,7 +1122,7 @@ def create_sliding_window_causal_mask( if sliding_window is None: raise ValueError("Could not find a `sliding_window` argument in the config, or it is not set") - batch_size, dtype = inputs_embeds.shape[0], inputs_embeds.dtype + batch_size, dtype, device = inputs_embeds.shape[0], inputs_embeds.dtype, inputs_embeds.device mask_factory_function = sliding_window_causal_mask_function(sliding_window) mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] @@ -1136,8 +1158,9 @@ def create_sliding_window_causal_mask( # We now create the mask causal_mask = mask_interface( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_factory_function, attention_mask=attention_mask, @@ -1146,6 +1169,7 @@ def create_sliding_window_causal_mask( dtype=dtype, # Additional kwarg for eager config=config, # Pass the config as well, in case someone wants to easily have their own mask_interface use_vmap=use_vmap, # Short-circuit to non-vmap expansions for the mask + device=device, ) return causal_mask @@ -1178,13 +1202,9 @@ def create_bidirectional_sliding_window_mask( An optional mask function to combine with the base mask function (by doing the intersection of both). This is useful to easily overlay another mask on top, for example for image tokens handling. """ - # Due to the logic surrounding `cache_position` in inferring query-related information, we - # construct a dummy tensor imitating initial positions - cache_position = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device, dtype=torch.long) - # We ignore a few irrelevant arguments at the end as we do not have a (growing) cache here - early_exit, attention_mask, _, kv_length, kv_offset = _preprocess_mask_arguments( - config, inputs_embeds, attention_mask, cache_position, None, None, 0 + early_exit, attention_mask, _, q_length, kv_length, q_offset, kv_offset = _preprocess_mask_arguments( + config, inputs_embeds, attention_mask, None, None, 0 ) if early_exit: return attention_mask @@ -1193,7 +1213,7 @@ def create_bidirectional_sliding_window_mask( if sliding_window is None: raise ValueError("Could not find a `sliding_window` argument in the config, or it is not set") - batch_size, dtype = inputs_embeds.shape[0], inputs_embeds.dtype + batch_size, dtype, device = inputs_embeds.shape[0], inputs_embeds.dtype, inputs_embeds.device mask_factory_function = sliding_window_bidirectional_mask_function(sliding_window) mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] @@ -1215,8 +1235,9 @@ def create_bidirectional_sliding_window_mask( attention_mask = mask_interface( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_factory_function, attention_mask=attention_mask, @@ -1226,6 +1247,7 @@ def create_bidirectional_sliding_window_mask( dtype=dtype, # Additional kwarg for eager config=config, # Pass the config as well, in case someone wants to easily have their own mask_interface use_vmap=use_vmap, # Short-circuit to non-vmap expansions for the mask + device=device, ) return attention_mask @@ -1235,7 +1257,7 @@ def create_chunked_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, + cache_position: torch.Tensor, # not used anymore but kept for BC past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -1257,7 +1279,7 @@ def create_chunked_causal_mask( The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. + Deprecated and unused. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) @@ -1275,8 +1297,8 @@ def create_chunked_causal_mask( else: layer_idx = 0 - early_exit, attention_mask, packed_sequence_mask, kv_length, kv_offset = _preprocess_mask_arguments( - config, inputs_embeds, attention_mask, cache_position, past_key_values, position_ids, layer_idx + early_exit, attention_mask, packed_sequence_mask, q_length, kv_length, q_offset, kv_offset = ( + _preprocess_mask_arguments(config, inputs_embeds, attention_mask, past_key_values, position_ids, layer_idx) ) if early_exit: return attention_mask @@ -1292,14 +1314,14 @@ def create_chunked_causal_mask( "chunked pattern cannot be respected. You should use another `attn_implementation` when instantiating the model" ) - batch_size, dtype = inputs_embeds.shape[0], inputs_embeds.dtype + batch_size, dtype, device = inputs_embeds.shape[0], inputs_embeds.dtype, inputs_embeds.device # For chunked attention and batched inputs, we need to take the number of left padding tokens into account # to start the chunk from the actual start of the sequence for the padded sequence if attention_mask is not None: # Only count the left padding tokens, not all of them left_padding_tokens = (attention_mask.cumsum(dim=-1) == torch.zeros_like(attention_mask)).sum(dim=-1) else: - left_padding_tokens = torch.zeros(batch_size, device=cache_position.device, dtype=int) + left_padding_tokens = torch.zeros(batch_size, device=device, dtype=int) mask_factory_function = chunked_causal_mask_function(chunk_size, left_padding_tokens) mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] @@ -1335,8 +1357,9 @@ def create_chunked_causal_mask( # We now create the mask causal_mask = mask_interface( batch_size=batch_size, - cache_position=cache_position, + q_length=q_length, kv_length=kv_length, + q_offset=q_offset, kv_offset=kv_offset, mask_function=mask_factory_function, attention_mask=attention_mask, @@ -1345,6 +1368,7 @@ def create_chunked_causal_mask( dtype=dtype, # Additional kwarg for eager config=config, # Pass the config as well, in case someone wants to easily have their own mask_interface use_vmap=use_vmap, # Short-circuit to non-vmap expansions for the mask + device=device, ) return causal_mask @@ -1361,7 +1385,6 @@ def create_masks_for_generate( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -1381,8 +1404,6 @@ def create_masks_for_generate( attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. - cache_position (`torch.Tensor`): - A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) @@ -1401,7 +1422,6 @@ def create_masks_for_generate( "config": effective_config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, "or_mask_function": or_mask_function, From b9672ff4a91eeabbf811ff6c315312795dad5671 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 17:27:21 +0100 Subject: [PATCH 02/42] fully remove from cache api --- src/transformers/cache_utils.py | 66 +++++++++------------------------ 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 7aee3e718fa8..0eb3e05012b7 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod from collections.abc import Iterable -from typing import Any import torch @@ -42,7 +41,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens @abstractmethod def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, cache_kwargs: dict[str, Any] | None = None + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: ... @abstractmethod @@ -97,10 +96,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self.is_initialized = True def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -108,7 +104,6 @@ def update( Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. - cache_kwargs (`dict[str, Any]`, *optional*): Additional arguments for the cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. @@ -183,10 +178,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self._sliding_window_tensor = self._sliding_window_tensor.to(self.device) def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -194,7 +186,6 @@ def update( Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. - cache_kwargs (`dict[str, Any]`, *optional*): Additional arguments for the cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. @@ -265,6 +256,7 @@ class StaticLayer(CacheLayerMixin): def __init__(self, max_cache_len: int): super().__init__() self.max_cache_len = max_cache_len + self.cumulative_length = 0 def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tensor) -> None: """ @@ -306,10 +298,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self.is_initialized = True def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -317,7 +306,6 @@ def update( Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. - cache_kwargs (`dict[str, Any]`, *optional*): Additional arguments for the cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. @@ -326,12 +314,10 @@ def update( if not self.is_initialized: self.lazy_initialization(key_states, value_states) - # Some old models give None for `cache_position` or even omit passing `cache_kwargs` when used as cross-attention, - # in which case we should copy the whole Layer (key_states.shape[-2] == self.max_cache_len) - cache_position = cache_kwargs.get("cache_position") if cache_kwargs is not None else None - cache_position = ( - cache_position if cache_position is not None else torch.arange(key_states.shape[-2], device=self.device) - ) + # Create a tensor to slice the static kv at the correct indices + kv_length = key_states.shape[-2] + cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length + self.cumulative_length += kv_length # Update the cache try: @@ -381,10 +367,7 @@ def __init__(self, max_cache_len: int, sliding_window: int): self.cumulative_length = 0 def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -392,7 +375,6 @@ def update( Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. - cache_kwargs (`dict[str, Any]`, *optional*): Additional arguments for the cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. @@ -401,17 +383,14 @@ def update( if not self.is_initialized: self.lazy_initialization(key_states, value_states) - # Some old models give None for `cache_position` or even omit passing `cache_kwargs` when used as cross-attention, - # in which case we should copy the whole Layer (key_states.shape[-2] == self.max_cache_len) - cache_position = cache_kwargs.get("cache_position") if cache_kwargs is not None else None - cache_position = ( - cache_position if cache_position is not None else torch.arange(key_states.shape[-2], device=self.device) - ) + # Create a tensor to slice the static kv at the correct indices + kv_length = key_states.shape[-2] + cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length cumulative_length = self.cumulative_length is_full = cumulative_length >= self.max_cache_len # Update it now that we saved the value above - self.cumulative_length += key_states.shape[-2] + self.cumulative_length += kv_length if is_full: # In general, we should use a much simpler `cat` here as well, independently of the states size. However, @@ -513,10 +492,7 @@ def __init__( self.cumulative_length = 0 def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -524,7 +500,6 @@ def update( Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. - cache_kwargs (`dict[str, Any]`, *optional*): Additional arguments for the cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. @@ -753,11 +728,7 @@ def offload(self, layer_idx: int, only_non_sliding: bool = True): self.layers[layer_idx].offload() def update( - self, - key_states: torch.Tensor, - value_states: torch.Tensor, - layer_idx: int, - cache_kwargs: dict[str, Any] | None = None, + self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. @@ -769,9 +740,6 @@ def update( The new value states to cache. layer_idx (`int`): The index of the layer to cache the states for. - cache_kwargs (`dict[str, Any]`, *optional*): - Additional arguments for the cache subclass. These are specific to each subclass and allow new types of - cache to be created. Return: A tuple containing the updated key and value states. @@ -786,7 +754,7 @@ def update( torch.cuda.default_stream(key_states.device).wait_stream(self.prefetch_stream) self.prefetch(layer_idx + 1, self.only_non_sliding) - keys, values = self.layers[layer_idx].update(key_states, value_states, cache_kwargs) + keys, values = self.layers[layer_idx].update(key_states, value_states, **kwargs) if self.offloading: self.offload(layer_idx, self.only_non_sliding) From 88a0b2ed2d481fb01d2e84317dcb57f835c14d2b Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 17:37:35 +0100 Subject: [PATCH 03/42] remove it from llama + modulars --- .../models/afmoe/modeling_afmoe.py | 2 -- .../models/apertus/modeling_apertus.py | 15 +++---------- .../models/arcee/modeling_arcee.py | 22 ++++--------------- src/transformers/models/aria/modeling_aria.py | 22 ++++--------------- .../models/bamba/modeling_bamba.py | 5 +---- .../models/bitnet/modeling_bitnet.py | 17 +++----------- .../models/cohere/modeling_cohere.py | 13 +++-------- src/transformers/models/csm/modeling_csm.py | 20 ++++------------- src/transformers/models/cwm/modeling_cwm.py | 4 ---- .../deepseek_v2/modeling_deepseek_v2.py | 17 +++----------- .../deepseek_v3/modeling_deepseek_v3.py | 17 +++----------- src/transformers/models/dia/modeling_dia.py | 5 +---- .../models/diffllama/modeling_diffllama.py | 17 +++----------- .../models/dots1/modeling_dots1.py | 4 ---- src/transformers/models/emu3/modeling_emu3.py | 20 ++++------------- .../models/ernie4_5/modeling_ernie4_5.py | 22 ++++--------------- .../ernie4_5_moe/modeling_ernie4_5_moe.py | 7 +----- .../modeling_ernie4_5_vl_moe.py | 5 +---- .../models/evolla/modeling_evolla.py | 5 +---- .../models/exaone4/modeling_exaone4.py | 1 - .../models/exaone_moe/modeling_exaone_moe.py | 3 --- .../models/gemma/modeling_gemma.py | 9 +------- src/transformers/models/glm/modeling_glm.py | 22 ++++--------------- src/transformers/models/glm4/modeling_glm4.py | 20 ++++------------- .../models/glm4_moe/modeling_glm4_moe.py | 17 +++----------- .../glm4_moe_lite/modeling_glm4_moe_lite.py | 17 +++----------- .../models/glm4v_moe/modeling_glm4v_moe.py | 2 -- .../glm_moe_dsa/modeling_glm_moe_dsa.py | 17 +++----------- .../models/granite/modeling_granite.py | 5 +---- .../models/granitemoe/modeling_granitemoe.py | 5 +---- .../modeling_granitemoeshared.py | 5 +---- .../models/helium/modeling_helium.py | 22 ++++--------------- .../modeling_hunyuan_v1_dense.py | 17 +++----------- .../hunyuan_v1_moe/modeling_hunyuan_v1_moe.py | 17 +++----------- .../models/jais2/modeling_jais2.py | 22 ++++--------------- .../modeling_kyutai_speech_to_text.py | 2 -- src/transformers/models/lfm2/modeling_lfm2.py | 2 -- .../models/lfm2_moe/modeling_lfm2_moe.py | 2 -- .../models/llama/modeling_llama.py | 22 ++++--------------- .../longcat_flash/modeling_longcat_flash.py | 2 -- .../models/ministral/modeling_ministral.py | 4 ---- .../models/ministral3/modeling_ministral3.py | 4 ---- .../models/mistral/modeling_mistral.py | 4 ---- .../models/moonshine/modeling_moonshine.py | 2 -- .../modeling_moonshine_streaming.py | 2 -- .../models/nanochat/modeling_nanochat.py | 2 -- src/transformers/models/olmo/modeling_olmo.py | 17 +++----------- .../models/olmo2/modeling_olmo2.py | 15 +++---------- .../models/olmo3/modeling_olmo3.py | 2 -- .../models/olmoe/modeling_olmoe.py | 2 -- .../paddleocr_vl/modeling_paddleocr_vl.py | 2 -- .../models/pe_audio/modeling_pe_audio.py | 2 -- .../pe_audio_video/modeling_pe_audio_video.py | 2 -- .../models/pe_video/modeling_pe_video.py | 2 -- src/transformers/models/phi/modeling_phi.py | 2 -- src/transformers/models/phi3/modeling_phi3.py | 2 -- .../models/phimoe/modeling_phimoe.py | 5 +---- .../models/qwen2/modeling_qwen2.py | 4 ---- .../models/qwen2_moe/modeling_qwen2_moe.py | 7 +----- .../models/qwen3/modeling_qwen3.py | 4 ---- .../models/qwen3_5/modeling_qwen3_5.py | 2 -- .../models/qwen3_moe/modeling_qwen3_moe.py | 2 -- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 6 ----- .../models/qwen3_vl/modeling_qwen3_vl.py | 1 - .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 -- .../models/seed_oss/modeling_seed_oss.py | 17 +++----------- .../models/smollm3/modeling_smollm3.py | 4 ---- .../models/solar_open/modeling_solar_open.py | 22 ++++--------------- .../models/starcoder2/modeling_starcoder2.py | 4 ---- .../models/youtu/modeling_youtu.py | 17 +++----------- 70 files changed, 100 insertions(+), 533 deletions(-) diff --git a/src/transformers/models/afmoe/modeling_afmoe.py b/src/transformers/models/afmoe/modeling_afmoe.py index 925b548af232..fe621838838e 100644 --- a/src/transformers/models/afmoe/modeling_afmoe.py +++ b/src/transformers/models/afmoe/modeling_afmoe.py @@ -657,7 +657,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -685,7 +684,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/apertus/modeling_apertus.py b/src/transformers/models/apertus/modeling_apertus.py index c900da4315a0..dc2d7329a186 100644 --- a/src/transformers/models/apertus/modeling_apertus.py +++ b/src/transformers/models/apertus/modeling_apertus.py @@ -377,7 +377,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -390,20 +389,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -419,7 +413,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -456,7 +449,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -489,7 +481,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/arcee/modeling_arcee.py b/src/transformers/models/arcee/modeling_arcee.py index 9cd48a09e468..8d2d05bf2952 100644 --- a/src/transformers/models/arcee/modeling_arcee.py +++ b/src/transformers/models/arcee/modeling_arcee.py @@ -250,7 +250,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -264,9 +263,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -306,7 +303,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -319,7 +315,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -380,7 +375,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -393,20 +387,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -422,7 +411,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -459,7 +447,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -487,7 +474,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 28685ed76d10..07d107a13d8b 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -479,7 +479,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -493,9 +492,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -546,7 +543,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -559,7 +555,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -714,7 +709,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -727,20 +721,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -756,7 +745,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -792,7 +780,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -820,7 +807,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/bamba/modeling_bamba.py b/src/transformers/models/bamba/modeling_bamba.py index 489eafded406..ace73fd37807 100644 --- a/src/transformers/models/bamba/modeling_bamba.py +++ b/src/transformers/models/bamba/modeling_bamba.py @@ -363,7 +363,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -377,9 +376,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/bitnet/modeling_bitnet.py b/src/transformers/models/bitnet/modeling_bitnet.py index 04a2aeb805e9..57eda03e5d57 100644 --- a/src/transformers/models/bitnet/modeling_bitnet.py +++ b/src/transformers/models/bitnet/modeling_bitnet.py @@ -239,7 +239,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -252,7 +251,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -378,7 +376,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -391,20 +388,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -420,7 +412,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -457,7 +448,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -490,7 +480,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/cohere/modeling_cohere.py b/src/transformers/models/cohere/modeling_cohere.py index 8b3725cac0c2..4128072a0c1c 100644 --- a/src/transformers/models/cohere/modeling_cohere.py +++ b/src/transformers/models/cohere/modeling_cohere.py @@ -412,7 +412,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -425,20 +424,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -454,7 +448,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/csm/modeling_csm.py b/src/transformers/models/csm/modeling_csm.py index 0759c7b7c745..3a13065728a0 100644 --- a/src/transformers/models/csm/modeling_csm.py +++ b/src/transformers/models/csm/modeling_csm.py @@ -301,7 +301,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -315,9 +314,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -357,7 +354,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -370,7 +366,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -704,7 +699,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -729,20 +723,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -758,7 +747,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/cwm/modeling_cwm.py b/src/transformers/models/cwm/modeling_cwm.py index 536e2f454b57..dc16b3ddea5f 100644 --- a/src/transformers/models/cwm/modeling_cwm.py +++ b/src/transformers/models/cwm/modeling_cwm.py @@ -297,7 +297,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -310,7 +309,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -462,7 +460,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -490,7 +487,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py index ea37bb77a1e9..e6618868c722 100644 --- a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py +++ b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py @@ -415,7 +415,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -428,7 +427,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -496,7 +494,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -509,20 +506,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -538,7 +530,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -575,7 +566,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -603,7 +593,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index edb88177760f..3fac50a92e24 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -504,7 +504,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -517,7 +516,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -590,7 +588,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -603,20 +600,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -632,7 +624,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -669,7 +660,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -697,7 +687,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/dia/modeling_dia.py b/src/transformers/models/dia/modeling_dia.py index f7ba0a3d4b81..8bd11629e675 100644 --- a/src/transformers/models/dia/modeling_dia.py +++ b/src/transformers/models/dia/modeling_dia.py @@ -298,7 +298,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -312,9 +311,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index 89d2eaae0c30..c03fcd902e2f 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -539,7 +539,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -552,7 +551,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -622,7 +620,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -635,20 +632,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -664,7 +656,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -701,7 +692,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -729,7 +719,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index c641dbf6a4e3..998fc7004b22 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -429,7 +429,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -442,7 +441,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -605,7 +603,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -638,7 +635,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index 46d662025f5e..deca1ef911d1 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -160,7 +160,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -174,9 +173,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1232,7 +1229,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -1245,20 +1241,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -1274,7 +1265,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1312,7 +1302,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -1341,7 +1330,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/ernie4_5/modeling_ernie4_5.py b/src/transformers/models/ernie4_5/modeling_ernie4_5.py index 8f2a7a146ab6..ae533c7c6ef8 100644 --- a/src/transformers/models/ernie4_5/modeling_ernie4_5.py +++ b/src/transformers/models/ernie4_5/modeling_ernie4_5.py @@ -227,7 +227,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -241,9 +240,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -304,7 +301,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -317,7 +313,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -378,7 +373,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -391,20 +385,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -420,7 +409,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -457,7 +445,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -474,7 +461,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py index b3e545776e02..24f05e3f5217 100644 --- a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py +++ b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py @@ -250,7 +250,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -264,9 +263,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -438,7 +435,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -451,7 +447,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index a0198139bc02..caa82d76c74d 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -235,7 +235,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -249,9 +248,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/evolla/modeling_evolla.py b/src/transformers/models/evolla/modeling_evolla.py index a3d3ea5ffbb5..238da9b2a0ff 100644 --- a/src/transformers/models/evolla/modeling_evolla.py +++ b/src/transformers/models/evolla/modeling_evolla.py @@ -1130,7 +1130,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -1144,9 +1143,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/exaone4/modeling_exaone4.py b/src/transformers/models/exaone4/modeling_exaone4.py index 4dd958af0138..a623e0f8d324 100644 --- a/src/transformers/models/exaone4/modeling_exaone4.py +++ b/src/transformers/models/exaone4/modeling_exaone4.py @@ -518,7 +518,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/exaone_moe/modeling_exaone_moe.py b/src/transformers/models/exaone_moe/modeling_exaone_moe.py index 35754ab51566..2f684c18e618 100644 --- a/src/transformers/models/exaone_moe/modeling_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modeling_exaone_moe.py @@ -355,7 +355,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -368,7 +367,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -641,7 +639,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py index be2b6361a53f..5f95e5612c77 100644 --- a/src/transformers/models/gemma/modeling_gemma.py +++ b/src/transformers/models/gemma/modeling_gemma.py @@ -250,7 +250,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -264,9 +263,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -306,7 +303,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -319,7 +315,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -472,7 +467,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -500,7 +494,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/glm/modeling_glm.py b/src/transformers/models/glm/modeling_glm.py index 0d36b3c30473..712202580943 100644 --- a/src/transformers/models/glm/modeling_glm.py +++ b/src/transformers/models/glm/modeling_glm.py @@ -244,7 +244,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -258,9 +257,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -321,7 +318,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -334,7 +330,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -395,7 +390,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -408,20 +402,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -437,7 +426,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -474,7 +462,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -502,7 +489,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/glm4/modeling_glm4.py b/src/transformers/models/glm4/modeling_glm4.py index 1f683a8bdb06..a9b00476e5e4 100644 --- a/src/transformers/models/glm4/modeling_glm4.py +++ b/src/transformers/models/glm4/modeling_glm4.py @@ -227,7 +227,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -241,9 +240,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -400,7 +397,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -413,20 +409,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -442,7 +433,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -479,7 +469,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | CausalLMOutputWithPast: @@ -512,7 +501,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/glm4_moe/modeling_glm4_moe.py b/src/transformers/models/glm4_moe/modeling_glm4_moe.py index b801ce679b9e..6b6c77ff4c72 100644 --- a/src/transformers/models/glm4_moe/modeling_glm4_moe.py +++ b/src/transformers/models/glm4_moe/modeling_glm4_moe.py @@ -447,7 +447,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -460,7 +459,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -533,7 +531,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -546,20 +543,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -575,7 +567,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -612,7 +603,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -640,7 +630,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py index b869228e0905..ed8cf6a478d8 100644 --- a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py +++ b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py @@ -521,7 +521,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -534,7 +533,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -607,7 +605,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -620,20 +617,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -649,7 +641,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -686,7 +677,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -714,7 +704,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 6edd4cedd4a4..2474100bc8d4 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -374,7 +374,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -387,7 +386,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py index a29dee5ff8c4..ab0286129710 100644 --- a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py @@ -604,7 +604,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -617,7 +616,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -761,7 +759,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -774,20 +771,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -803,7 +795,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -840,7 +831,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -868,7 +858,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granite/modeling_granite.py b/src/transformers/models/granite/modeling_granite.py index 1541f6cad55d..678b4df53a2f 100644 --- a/src/transformers/models/granite/modeling_granite.py +++ b/src/transformers/models/granite/modeling_granite.py @@ -147,7 +147,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -161,9 +160,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/granitemoe/modeling_granitemoe.py b/src/transformers/models/granitemoe/modeling_granitemoe.py index e6d98911f362..aea9b676c95c 100644 --- a/src/transformers/models/granitemoe/modeling_granitemoe.py +++ b/src/transformers/models/granitemoe/modeling_granitemoe.py @@ -369,7 +369,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -383,9 +382,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py index 91f6a4ed5158..8a3536df212a 100644 --- a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py +++ b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py @@ -357,7 +357,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -371,9 +370,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/helium/modeling_helium.py b/src/transformers/models/helium/modeling_helium.py index 1ce658554f13..8283fcb19e28 100644 --- a/src/transformers/models/helium/modeling_helium.py +++ b/src/transformers/models/helium/modeling_helium.py @@ -249,7 +249,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -263,9 +262,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -305,7 +302,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -318,7 +314,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -379,7 +374,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -392,20 +386,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -421,7 +410,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -458,7 +446,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -486,7 +473,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py index a6ad370e1dd9..9be66c29776a 100644 --- a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py +++ b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py @@ -245,7 +245,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -258,7 +257,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -417,7 +415,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -430,20 +427,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -459,7 +451,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -496,7 +487,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -524,7 +514,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py index 43836cd05bc5..a7043296afd1 100644 --- a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py +++ b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py @@ -332,7 +332,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -345,7 +344,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -506,7 +504,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -519,20 +516,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -548,7 +540,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -585,7 +576,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -613,7 +603,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/jais2/modeling_jais2.py b/src/transformers/models/jais2/modeling_jais2.py index bbb039194041..5e6a37c0172d 100644 --- a/src/transformers/models/jais2/modeling_jais2.py +++ b/src/transformers/models/jais2/modeling_jais2.py @@ -158,7 +158,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -172,9 +171,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -214,7 +211,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -227,7 +223,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -353,7 +348,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -366,20 +360,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -395,7 +384,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -432,7 +420,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -460,7 +447,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py index 6253c6aacfa1..95accc95bf3f 100644 --- a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py +++ b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py @@ -947,7 +947,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -990,7 +989,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/lfm2/modeling_lfm2.py b/src/transformers/models/lfm2/modeling_lfm2.py index b914560599b8..db24d85ad0ca 100644 --- a/src/transformers/models/lfm2/modeling_lfm2.py +++ b/src/transformers/models/lfm2/modeling_lfm2.py @@ -719,7 +719,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -747,7 +746,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py index 95f7716da62b..a659342bc53a 100644 --- a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py @@ -809,7 +809,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -837,7 +836,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index b1bd01cc32f3..9d659c7c6f08 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -254,7 +254,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -268,9 +267,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -310,7 +307,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -323,7 +319,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -384,7 +379,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -397,20 +391,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -426,7 +415,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -463,7 +451,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -491,7 +478,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/longcat_flash/modeling_longcat_flash.py b/src/transformers/models/longcat_flash/modeling_longcat_flash.py index 77e550503131..4f972fa88e5e 100644 --- a/src/transformers/models/longcat_flash/modeling_longcat_flash.py +++ b/src/transformers/models/longcat_flash/modeling_longcat_flash.py @@ -683,7 +683,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -711,7 +710,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/ministral/modeling_ministral.py b/src/transformers/models/ministral/modeling_ministral.py index 90634d3973c2..d66c5ff04371 100644 --- a/src/transformers/models/ministral/modeling_ministral.py +++ b/src/transformers/models/ministral/modeling_ministral.py @@ -224,7 +224,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -237,7 +236,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -450,7 +448,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -478,7 +475,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py index 1c26ab3a0aec..646904a06e74 100644 --- a/src/transformers/models/ministral3/modeling_ministral3.py +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -228,7 +228,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -241,7 +240,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -446,7 +444,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -474,7 +471,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py index a2ec7f67c262..1f457ecf0695 100644 --- a/src/transformers/models/mistral/modeling_mistral.py +++ b/src/transformers/models/mistral/modeling_mistral.py @@ -218,7 +218,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -231,7 +230,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -436,7 +434,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -464,7 +461,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/moonshine/modeling_moonshine.py b/src/transformers/models/moonshine/modeling_moonshine.py index 4345d3634da8..b48b25096a07 100644 --- a/src/transformers/models/moonshine/modeling_moonshine.py +++ b/src/transformers/models/moonshine/modeling_moonshine.py @@ -393,7 +393,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -406,7 +405,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py index a58780b73252..167adb7f6fcf 100644 --- a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py +++ b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py @@ -247,7 +247,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -260,7 +259,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/nanochat/modeling_nanochat.py b/src/transformers/models/nanochat/modeling_nanochat.py index cac6b9afa60f..e680b3cbaddd 100644 --- a/src/transformers/models/nanochat/modeling_nanochat.py +++ b/src/transformers/models/nanochat/modeling_nanochat.py @@ -304,7 +304,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -317,7 +316,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/olmo/modeling_olmo.py b/src/transformers/models/olmo/modeling_olmo.py index dcc3a6b61526..df73ff2b7e6a 100644 --- a/src/transformers/models/olmo/modeling_olmo.py +++ b/src/transformers/models/olmo/modeling_olmo.py @@ -307,7 +307,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -320,7 +319,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -381,7 +379,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -394,20 +391,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -423,7 +415,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -460,7 +451,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -488,7 +478,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmo2/modeling_olmo2.py b/src/transformers/models/olmo2/modeling_olmo2.py index 67516a34ffbc..465af484bf63 100644 --- a/src/transformers/models/olmo2/modeling_olmo2.py +++ b/src/transformers/models/olmo2/modeling_olmo2.py @@ -385,7 +385,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -398,20 +397,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -427,7 +421,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -464,7 +457,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -492,7 +484,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmo3/modeling_olmo3.py b/src/transformers/models/olmo3/modeling_olmo3.py index f85d4aa361d7..23423be782aa 100644 --- a/src/transformers/models/olmo3/modeling_olmo3.py +++ b/src/transformers/models/olmo3/modeling_olmo3.py @@ -471,7 +471,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -499,7 +498,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index ff367f1ab4f7..93b17496d111 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -394,7 +394,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -407,7 +406,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index cd616538725c..b0774abea9f0 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -401,7 +401,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -414,7 +413,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/pe_audio/modeling_pe_audio.py b/src/transformers/models/pe_audio/modeling_pe_audio.py index 5f58130e2146..e502073a95c7 100644 --- a/src/transformers/models/pe_audio/modeling_pe_audio.py +++ b/src/transformers/models/pe_audio/modeling_pe_audio.py @@ -466,7 +466,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -479,7 +478,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py index a76e1b40f27c..e840baecd41b 100644 --- a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py +++ b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py @@ -402,7 +402,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -415,7 +414,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index fbc32ed0983d..db0377f73c68 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -370,7 +370,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -383,7 +382,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/phi/modeling_phi.py b/src/transformers/models/phi/modeling_phi.py index cdf24e069c77..2f4906a61d29 100644 --- a/src/transformers/models/phi/modeling_phi.py +++ b/src/transformers/models/phi/modeling_phi.py @@ -484,7 +484,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -512,7 +511,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/phi3/modeling_phi3.py b/src/transformers/models/phi3/modeling_phi3.py index 3e0bdef972ca..3de9c5edd3d9 100644 --- a/src/transformers/models/phi3/modeling_phi3.py +++ b/src/transformers/models/phi3/modeling_phi3.py @@ -467,7 +467,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -495,7 +494,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/phimoe/modeling_phimoe.py b/src/transformers/models/phimoe/modeling_phimoe.py index 8d88f3b041c5..d93e8bf7cee7 100644 --- a/src/transformers/models/phimoe/modeling_phimoe.py +++ b/src/transformers/models/phimoe/modeling_phimoe.py @@ -224,7 +224,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -238,9 +237,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/qwen2/modeling_qwen2.py b/src/transformers/models/qwen2/modeling_qwen2.py index 8c55adfa5db6..446680dae473 100644 --- a/src/transformers/models/qwen2/modeling_qwen2.py +++ b/src/transformers/models/qwen2/modeling_qwen2.py @@ -288,7 +288,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -301,7 +300,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -452,7 +450,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -480,7 +477,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index d53d9ae6ff32..b88b8fcc0bed 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -256,7 +256,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -270,9 +269,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -398,7 +395,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -411,7 +407,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3/modeling_qwen3.py b/src/transformers/models/qwen3/modeling_qwen3.py index c5a438ec96e8..5db2cc38df97 100644 --- a/src/transformers/models/qwen3/modeling_qwen3.py +++ b/src/transformers/models/qwen3/modeling_qwen3.py @@ -313,7 +313,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -326,7 +325,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -477,7 +475,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -510,7 +507,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_5/modeling_qwen3_5.py b/src/transformers/models/qwen3_5/modeling_qwen3_5.py index 96efb5838ca4..8fb864689a2c 100644 --- a/src/transformers/models/qwen3_5/modeling_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modeling_qwen3_5.py @@ -1735,7 +1735,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -1768,7 +1767,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index b07006597a66..025cb3a8de52 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -331,7 +331,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -344,7 +343,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 6bc957e21139..95eeca6908fb 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1581,7 +1581,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -1594,7 +1593,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -2473,7 +2471,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -2486,7 +2483,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -2893,7 +2889,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -2906,7 +2901,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index b09494204048..0208a24ec05d 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -547,7 +547,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index dc2079582c78..d163e85ae96d 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -328,7 +328,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -341,7 +340,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/seed_oss/modeling_seed_oss.py b/src/transformers/models/seed_oss/modeling_seed_oss.py index 1cccc13d221b..c73268097848 100644 --- a/src/transformers/models/seed_oss/modeling_seed_oss.py +++ b/src/transformers/models/seed_oss/modeling_seed_oss.py @@ -246,7 +246,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -259,7 +258,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -385,7 +383,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -398,20 +395,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -427,7 +419,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -464,7 +455,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -497,7 +487,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/smollm3/modeling_smollm3.py b/src/transformers/models/smollm3/modeling_smollm3.py index 1a771ccfccef..acb7a02e87ee 100644 --- a/src/transformers/models/smollm3/modeling_smollm3.py +++ b/src/transformers/models/smollm3/modeling_smollm3.py @@ -316,7 +316,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -329,7 +328,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -480,7 +478,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -508,7 +505,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/solar_open/modeling_solar_open.py b/src/transformers/models/solar_open/modeling_solar_open.py index 37745453b35a..4be8c5c85288 100644 --- a/src/transformers/models/solar_open/modeling_solar_open.py +++ b/src/transformers/models/solar_open/modeling_solar_open.py @@ -64,7 +64,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -77,7 +76,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -324,7 +322,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -338,9 +335,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -509,7 +504,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -522,20 +516,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -551,7 +540,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -588,7 +576,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -616,7 +603,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/starcoder2/modeling_starcoder2.py b/src/transformers/models/starcoder2/modeling_starcoder2.py index bd97e1c9ee91..503453e7db1c 100644 --- a/src/transformers/models/starcoder2/modeling_starcoder2.py +++ b/src/transformers/models/starcoder2/modeling_starcoder2.py @@ -221,7 +221,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -234,7 +233,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -444,7 +442,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -472,7 +469,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/youtu/modeling_youtu.py b/src/transformers/models/youtu/modeling_youtu.py index a247b62f9ec7..59026df304da 100644 --- a/src/transformers/models/youtu/modeling_youtu.py +++ b/src/transformers/models/youtu/modeling_youtu.py @@ -404,7 +404,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -417,7 +416,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -488,7 +486,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -501,20 +498,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -530,7 +522,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -567,7 +558,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -595,7 +585,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) From f5dc06cd2519edec66318574d0607333be4a9bc3 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 19:00:30 +0100 Subject: [PATCH 04/42] remove them from all modulars of llama descendants --- .../models/afmoe/modular_afmoe.py | 20 ++------- .../models/apertus/modular_apertus.py | 6 +-- src/transformers/models/aria/modular_aria.py | 6 --- .../audioflamingo3/modular_audioflamingo3.py | 7 +--- .../models/bitnet/modular_bitnet.py | 5 +-- .../models/cohere/modular_cohere.py | 11 +---- src/transformers/models/cwm/modular_cwm.py | 13 ++---- .../models/deepseek_v2/modular_deepseek_v2.py | 5 +-- .../models/deepseek_v3/modular_deepseek_v3.py | 5 +-- src/transformers/models/dia/modular_dia.py | 17 ++------ .../models/diffllama/modular_diffllama.py | 15 ++----- src/transformers/models/emu3/modular_emu3.py | 8 ---- .../modular_ernie4_5_vl_moe.py | 23 ++--------- .../models/evolla/modular_evolla.py | 15 ++----- .../models/exaone4/modular_exaone4.py | 21 ++-------- .../models/exaone_moe/modular_exaone_moe.py | 2 - .../models/gemma/modular_gemma.py | 13 ++---- src/transformers/models/glm4/modular_glm4.py | 2 - .../models/glm4v_moe/modeling_glm4v_moe.py | 26 +++--------- .../models/glm_moe_dsa/modular_glm_moe_dsa.py | 4 +- .../models/glmasr/modular_glmasr.py | 2 - .../models/granite/modular_granite.py | 19 ++------- .../models/granitemoe/modular_granitemoe.py | 16 ++------ .../modular_granitemoeshared.py | 2 - .../modular_hunyuan_v1_dense.py | 5 +-- .../hunyuan_v1_moe/modular_hunyuan_v1_moe.py | 5 +-- src/transformers/models/lfm2/modular_lfm2.py | 35 +++++----------- .../models/lfm2_moe/modular_lfm2_moe.py | 13 ++---- .../longcat_flash/modular_longcat_flash.py | 21 ++-------- .../models/ministral/modular_ministral.py | 13 ++---- .../models/ministral3/modular_ministral3.py | 7 ++-- .../models/mistral/modular_mistral.py | 18 ++------ .../models/moonshine/modular_moonshine.py | 29 +++---------- .../modular_moonshine_streaming.py | 2 - .../models/nanochat/modular_nanochat.py | 18 ++------ src/transformers/models/olmo/modular_olmo.py | 5 +-- .../models/olmo2/modular_olmo2.py | 7 +--- .../models/olmo3/modular_olmo3.py | 18 ++------ .../models/olmoe/modular_olmoe.py | 17 ++------ .../paddleocr_vl/modular_paddleocr_vl.py | 17 ++------ src/transformers/models/phi/modular_phi.py | 20 ++------- src/transformers/models/phi3/modular_phi3.py | 11 +---- .../models/phimoe/modular_phimoe.py | 4 +- .../models/qwen2/modular_qwen2.py | 18 ++------ .../models/qwen2_moe/modular_qwen2_moe.py | 13 ++---- .../models/qwen3/modular_qwen3.py | 5 +-- .../models/qwen3_5/modular_qwen3_5.py | 28 +++---------- .../models/qwen3_moe/modular_qwen3_moe.py | 2 - .../models/qwen3_next/modular_qwen3_next.py | 41 +++++-------------- .../qwen3_omni_moe/modular_qwen3_omni_moe.py | 27 ++---------- .../models/qwen3_vl/modular_qwen3_vl.py | 26 ++---------- .../qwen3_vl_moe/modular_qwen3_vl_moe.py | 15 ++----- .../models/seed_oss/modular_seed_oss.py | 5 +-- .../models/smollm3/modular_smollm3.py | 4 +- .../models/starcoder2/modular_starcoder2.py | 18 ++------ 55 files changed, 144 insertions(+), 586 deletions(-) diff --git a/src/transformers/models/afmoe/modular_afmoe.py b/src/transformers/models/afmoe/modular_afmoe.py index c5e7eb1faef1..5fd37eb57e07 100644 --- a/src/transformers/models/afmoe/modular_afmoe.py +++ b/src/transformers/models/afmoe/modular_afmoe.py @@ -204,7 +204,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_value: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -224,8 +223,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_value is not None: - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -287,7 +285,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_value: Cache | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: @@ -301,7 +298,6 @@ def forward( position_ids=position_ids, past_key_value=past_key_value, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -391,7 +387,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: @@ -404,15 +399,10 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, - past_seen_tokens + inputs_embeds.shape[1], - device=inputs_embeds.device, - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -420,7 +410,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, } causal_mask_mapping = { @@ -443,7 +432,6 @@ def forward( position_ids=position_ids, past_key_value=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/apertus/modular_apertus.py b/src/transformers/models/apertus/modular_apertus.py index 6caebcd27666..47a344a8c3fd 100644 --- a/src/transformers/models/apertus/modular_apertus.py +++ b/src/transformers/models/apertus/modular_apertus.py @@ -216,7 +216,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -232,8 +231,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -271,7 +269,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor]: @@ -283,7 +280,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/aria/modular_aria.py b/src/transformers/models/aria/modular_aria.py index 07bf00d04569..a23da3b6edb1 100644 --- a/src/transformers/models/aria/modular_aria.py +++ b/src/transformers/models/aria/modular_aria.py @@ -1295,7 +1295,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple | AriaModelOutputWithPast: if inputs_embeds is None: @@ -1321,7 +1320,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1374,7 +1372,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, logits_to_keep: int | torch.Tensor = 0, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | AriaCausalLMOutputWithPast: r""" @@ -1447,7 +1444,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1478,7 +1474,6 @@ def prepare_inputs_for_generation( pixel_values=None, pixel_mask=None, attention_mask=None, - cache_position=None, logits_to_keep=None, is_first_iteration=False, **kwargs, @@ -1488,7 +1483,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, logits_to_keep=logits_to_keep, is_first_iteration=is_first_iteration, **kwargs, diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 4efa10c7f702..7e97fd5edcee 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -197,7 +197,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -283,22 +282,20 @@ def forward( past_key_values=past_key_values, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) return outputs - def prepare_inputs_for_generation(self, *args, **kwargs): + def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, **kwargs): # Overwritten -- we should not pass input_features when we are in cached decoding stage input_features = kwargs.pop("input_features", None) input_features_mask = kwargs.pop("input_features_mask", None) - cache_position = kwargs.get("cache_position") model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if cache_position is not None and model_inputs["cache_position"][0] == 0: + if is_first_iteration: # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features diff --git a/src/transformers/models/bitnet/modular_bitnet.py b/src/transformers/models/bitnet/modular_bitnet.py index e00d14fbc897..7f3a248766b2 100644 --- a/src/transformers/models/bitnet/modular_bitnet.py +++ b/src/transformers/models/bitnet/modular_bitnet.py @@ -63,7 +63,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -77,9 +76,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/cohere/modular_cohere.py b/src/transformers/models/cohere/modular_cohere.py index 3c215a77780f..eafd9183d376 100644 --- a/src/transformers/models/cohere/modular_cohere.py +++ b/src/transformers/models/cohere/modular_cohere.py @@ -148,7 +148,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -170,9 +169,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; position_ids needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -209,7 +206,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -226,8 +222,6 @@ def forward( use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. @@ -241,7 +235,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -278,7 +271,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -319,7 +311,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/cwm/modular_cwm.py b/src/transformers/models/cwm/modular_cwm.py index b877bae88e46..3f99752a9d60 100644 --- a/src/transformers/models/cwm/modular_cwm.py +++ b/src/transformers/models/cwm/modular_cwm.py @@ -224,7 +224,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> CwmModelOutputWithPast: @@ -237,21 +236,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) if not isinstance(causal_mask_mapping := attention_mask, dict): mask_kwargs = { "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -271,7 +265,6 @@ def forward( attention_mask=causal_mask_mapping[decoder_layer.attention_type], position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/deepseek_v2/modular_deepseek_v2.py b/src/transformers/models/deepseek_v2/modular_deepseek_v2.py index 30e9c91116e2..496ff9a545d2 100644 --- a/src/transformers/models/deepseek_v2/modular_deepseek_v2.py +++ b/src/transformers/models/deepseek_v2/modular_deepseek_v2.py @@ -363,7 +363,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: @@ -391,9 +390,7 @@ def forward( key_states = torch.cat((k_nope, k_pe), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) diff --git a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py index 2fddcfc1f91b..3c62a564a31d 100644 --- a/src/transformers/models/deepseek_v3/modular_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modular_deepseek_v3.py @@ -224,7 +224,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -257,9 +256,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) diff --git a/src/transformers/models/dia/modular_dia.py b/src/transformers/models/dia/modular_dia.py index cbf74517e61c..005a2bc506f3 100644 --- a/src/transformers/models/dia/modular_dia.py +++ b/src/transformers/models/dia/modular_dia.py @@ -320,7 +320,6 @@ def forward( encoder_hidden_states: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, past_key_values: EncoderDecoderCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: self_attn_cache = past_key_values @@ -336,7 +335,6 @@ def forward( # Needs to be an arg in order to function properly # on inplace operations to be carried (e.g. compile) self_attn_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + self_attn_output @@ -388,7 +386,6 @@ def forward( past_key_values: EncoderDecoderCache | None = None, output_attentions: bool | None = False, output_hidden_states: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> BaseModelOutputWithPastAndCrossAttentions | tuple: r""" @@ -400,12 +397,10 @@ def forward( batch_size, seq_length = input_ids.size()[:-1] past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 0 - if cache_position is None: - cache_position = torch.arange( - past_key_values_length, past_key_values_length + seq_length, device=input_ids.device - ) + if position_ids is None: - position_ids = cache_position[None, :] + position_ids = torch.arange(seq_length, device=input_ids.device) + past_key_values_length + position_ids = position_ids.unsqueeze(0) # RoPE hidden_states = self.embeddings(input_ids) @@ -419,7 +414,6 @@ def forward( config=self.config, inputs_embeds=hidden_states, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, ) encoder_attention_mask = create_bidirectional_mask( @@ -447,7 +441,6 @@ def forward( encoder_hidden_states, encoder_attention_mask=encoder_attention_mask, past_key_values=past_key_values, - cache_position=cache_position, position_ids=position_ids, **kwargs, ) @@ -500,7 +493,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | Seq2SeqModelOutput: r""" @@ -582,7 +574,6 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -637,7 +628,6 @@ def forward( output_attentions: bool | None = None, output_hidden_states: bool | None = None, labels: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | Seq2SeqLMOutput: r""" @@ -677,7 +667,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/diffllama/modular_diffllama.py b/src/transformers/models/diffllama/modular_diffllama.py index 897a436cb31f..71087303c97e 100644 --- a/src/transformers/models/diffllama/modular_diffllama.py +++ b/src/transformers/models/diffllama/modular_diffllama.py @@ -102,7 +102,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: bsz, target_len, _ = hidden_states.size() @@ -120,9 +119,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) @@ -179,7 +176,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, ) -> tuple[torch.Tensor, None]: if isinstance(past_key_values, StaticCache): raise ValueError( @@ -204,9 +200,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache # to be able to avoid many of these transpose/reshape/view. @@ -307,7 +301,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: bsz, q_len, _ = hidden_states.size() @@ -324,9 +317,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) diff --git a/src/transformers/models/emu3/modular_emu3.py b/src/transformers/models/emu3/modular_emu3.py index e3c64ab770c1..ec384c518303 100644 --- a/src/transformers/models/emu3/modular_emu3.py +++ b/src/transformers/models/emu3/modular_emu3.py @@ -70,7 +70,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -83,7 +82,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1037,7 +1035,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | CausalLMOutputWithPast: r""" @@ -1069,7 +1066,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1116,7 +1112,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, labels: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], @@ -1176,7 +1171,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1205,7 +1199,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1219,7 +1212,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, use_cache=use_cache, diff --git a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py index 68a181eccfa3..0a6696b1df2b 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py @@ -669,7 +669,6 @@ def forward( position_ids: torch.Tensor | None = None, moe_mm_token_type_ids: torch.IntTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: residual = hidden_states @@ -683,7 +682,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = hidden_states + residual @@ -759,7 +757,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> MoeModelOutputWithPast: r""" @@ -775,15 +772,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -804,7 +797,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -822,7 +814,6 @@ def forward( position_ids=position_ids, moe_mm_token_type_ids=moe_mm_token_type_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) @@ -1245,7 +1236,7 @@ def compute_3d_position_ids( delta = self.rope_deltas.repeat_interleave(batch_size // self.rope_deltas.shape[0], dim=0) position_ids = (position_ids + delta).to(device=inputs_embeds.device) else: - # Can't build correct 3D positions. Let the model infer it from `cache_position` + # Can't build correct 3D positions. Let the model infer it position_ids = None return position_ids @@ -1266,7 +1257,6 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: r""" @@ -1320,7 +1310,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, return_dict=True, - cache_position=cache_position, **kwargs, ) @@ -1356,7 +1345,6 @@ def prepare_inputs_for_generation( input_ids, inputs_embeds=None, attention_mask=None, - cache_position=None, past_key_values=None, image_grid_thw=None, video_grid_thw=None, @@ -1369,7 +1357,6 @@ def prepare_inputs_for_generation( input_ids, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, @@ -1406,7 +1393,6 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeCausalLMOutputWithPast: @@ -1446,7 +1432,6 @@ def forward( image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, rope_deltas=rope_deltas, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/evolla/modular_evolla.py b/src/transformers/models/evolla/modular_evolla.py index 6a6c67639eaf..9e0deb0566c3 100644 --- a/src/transformers/models/evolla/modular_evolla.py +++ b/src/transformers/models/evolla/modular_evolla.py @@ -668,7 +668,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, protein_kv_states: torch.Tensor | None = None, structure_kv_states: torch.Tensor | None = None, msa_kv_states: torch.Tensor | None = None, @@ -689,7 +688,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -777,7 +775,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, protein_input_ids: torch.LongTensor | None = None, protein_attention_mask: torch.Tensor | None = None, structure_feats: torch.FloatTensor | None = None, @@ -809,14 +806,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) protein_feats = None protein_batch_mask = None @@ -837,7 +830,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, ) @@ -851,7 +843,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, protein_kv_states=protein_feats, structure_kv_states=structure_feats, msa_kv_states=msa_feats, diff --git a/src/transformers/models/exaone4/modular_exaone4.py b/src/transformers/models/exaone4/modular_exaone4.py index 6c8f98a5cb57..0a3e51f125aa 100644 --- a/src/transformers/models/exaone4/modular_exaone4.py +++ b/src/transformers/models/exaone4/modular_exaone4.py @@ -260,7 +260,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -280,10 +279,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = { - "cache_position": cache_position, - } - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -340,7 +336,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -352,14 +347,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -368,7 +359,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -390,7 +380,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -413,7 +402,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -456,7 +444,6 @@ def forward( inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) diff --git a/src/transformers/models/exaone_moe/modular_exaone_moe.py b/src/transformers/models/exaone_moe/modular_exaone_moe.py index 7e323421f550..d407fa7141f6 100644 --- a/src/transformers/models/exaone_moe/modular_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modular_exaone_moe.py @@ -314,7 +314,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -357,7 +356,6 @@ def forward( inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) diff --git a/src/transformers/models/gemma/modular_gemma.py b/src/transformers/models/gemma/modular_gemma.py index f59f1bb3c211..06005080af50 100644 --- a/src/transformers/models/gemma/modular_gemma.py +++ b/src/transformers/models/gemma/modular_gemma.py @@ -243,7 +243,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -255,20 +254,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -290,7 +284,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/glm4/modular_glm4.py b/src/transformers/models/glm4/modular_glm4.py index b9ec811fbc3d..4921ac3172d0 100644 --- a/src/transformers/models/glm4/modular_glm4.py +++ b/src/transformers/models/glm4/modular_glm4.py @@ -55,7 +55,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -68,7 +67,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 2474100bc8d4..32678a2dcd4d 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -157,7 +157,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -175,9 +174,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; position_ids needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -993,7 +990,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -1006,15 +1002,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -1039,7 +1031,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": text_position_ids, } @@ -1058,7 +1049,6 @@ def forward( attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = layer_outputs @@ -1396,7 +1386,7 @@ def compute_3d_position_ids( delta = self.rope_deltas.repeat_interleave(batch_size // self.rope_deltas.shape[0], dim=0) position_ids = position_ids + delta.to(device=position_ids.device) else: - # Can't build correct 3D positions. Let the model infer it from `cache_position` + # Can't build correct 3D positions. Let the model infer it position_ids = None return position_ids @@ -1414,7 +1404,6 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Glm4vMoeModelOutputWithPast: r""" @@ -1459,7 +1448,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1618,7 +1606,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Glm4vMoeCausalLMOutputWithPast: @@ -1674,7 +1661,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1717,7 +1703,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1734,7 +1719,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py index 65a986013669..8589d55b4de8 100644 --- a/src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py @@ -497,7 +497,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -538,8 +537,7 @@ def forward( # Cache update if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # ===== Indexer (DSA sparse mask) ===== # attention_mask is [B, 1, S, T] (4D) for eager and (2D) otherwise but indexer works with [B, S, T] (3D) diff --git a/src/transformers/models/glmasr/modular_glmasr.py b/src/transformers/models/glmasr/modular_glmasr.py index 037c0c925995..671e4881d9ce 100644 --- a/src/transformers/models/glmasr/modular_glmasr.py +++ b/src/transformers/models/glmasr/modular_glmasr.py @@ -396,7 +396,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -436,7 +435,6 @@ def forward( inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) diff --git a/src/transformers/models/granite/modular_granite.py b/src/transformers/models/granite/modular_granite.py index a4e979fb2324..833e3b4af4ee 100644 --- a/src/transformers/models/granite/modular_granite.py +++ b/src/transformers/models/granite/modular_granite.py @@ -56,7 +56,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -73,8 +72,6 @@ def forward( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Cache`, *optional*): cached past key and value projection states - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. @@ -94,7 +91,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -136,7 +132,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -162,20 +157,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -198,7 +188,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -234,7 +223,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -253,7 +241,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoe/modular_granitemoe.py b/src/transformers/models/granitemoe/modular_granitemoe.py index 88c50171096e..1ea37a919b49 100644 --- a/src/transformers/models/granitemoe/modular_granitemoe.py +++ b/src/transformers/models/granitemoe/modular_granitemoe.py @@ -114,7 +114,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> torch.Tensor: @@ -124,7 +123,6 @@ def forward( hidden_states=hidden_states, attention_mask=attention_mask, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -175,7 +173,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -187,19 +184,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( # ONLY DIFF WITH MIXTRAL: NO SLIDING config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -217,7 +210,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -246,7 +238,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | MoeCausalLMOutputWithPast: @@ -282,7 +273,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoeshared/modular_granitemoeshared.py b/src/transformers/models/granitemoeshared/modular_granitemoeshared.py index efb03ad06a87..e51cd7712b9b 100644 --- a/src/transformers/models/granitemoeshared/modular_granitemoeshared.py +++ b/src/transformers/models/granitemoeshared/modular_granitemoeshared.py @@ -96,7 +96,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[GraniteFlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -111,7 +110,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/hunyuan_v1_dense/modular_hunyuan_v1_dense.py b/src/transformers/models/hunyuan_v1_dense/modular_hunyuan_v1_dense.py index 791791585b78..cccad38a7a14 100644 --- a/src/transformers/models/hunyuan_v1_dense/modular_hunyuan_v1_dense.py +++ b/src/transformers/models/hunyuan_v1_dense/modular_hunyuan_v1_dense.py @@ -72,7 +72,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -88,9 +87,7 @@ def forward( key_states = self.key_layernorm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/hunyuan_v1_moe/modular_hunyuan_v1_moe.py b/src/transformers/models/hunyuan_v1_moe/modular_hunyuan_v1_moe.py index 54c85fb02ef2..2a60c26bc391 100644 --- a/src/transformers/models/hunyuan_v1_moe/modular_hunyuan_v1_moe.py +++ b/src/transformers/models/hunyuan_v1_moe/modular_hunyuan_v1_moe.py @@ -69,7 +69,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -85,9 +84,7 @@ def forward( key_states = self.key_layernorm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/lfm2/modular_lfm2.py b/src/transformers/models/lfm2/modular_lfm2.py index 649e63b3d0f5..ff0f66554024 100644 --- a/src/transformers/models/lfm2/modular_lfm2.py +++ b/src/transformers/models/lfm2/modular_lfm2.py @@ -179,7 +179,7 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. @@ -187,7 +187,6 @@ def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[ for each layer. """ full_mask_kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length() kv_length = query_length + past_seen_tokens return kv_length, full_mask_kv_offset @@ -232,7 +231,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -246,8 +244,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -295,7 +292,6 @@ def cuda_kernels_forward( self, x: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): x = apply_mask_to_padding_states(x, attention_mask) @@ -305,7 +301,7 @@ def cuda_kernels_forward( Bx = B * x conv_weights = self.conv.weight.view(self.conv.weight.size(0), self.conv.weight.size(2)) - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_out = causal_conv1d_update( Bx.squeeze(-1), past_key_values.conv_cache[self.layer_idx], @@ -329,7 +325,6 @@ def slow_forward( self, x: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): seqlen = x.shape[1] @@ -340,8 +335,9 @@ def slow_forward( Bx = B * x - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] + cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) @@ -367,12 +363,11 @@ def forward( self, hidden_states: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): if is_fast_path_available and "cuda" in hidden_states.device.type and not is_torchdynamo_compiling(): - return self.cuda_kernels_forward(hidden_states, past_key_values, cache_position, attention_mask) - return self.slow_forward(hidden_states, past_key_values, cache_position, attention_mask) + return self.cuda_kernels_forward(hidden_states, past_key_values, attention_mask) + return self.slow_forward(hidden_states, past_key_values, attention_mask) class Lfm2DecoderLayer(GradientCheckpointingLayer): @@ -395,7 +390,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> torch.Tensor: residual = hidden_states @@ -406,14 +400,12 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) else: hidden_states = self.conv( hidden_states=self.operator_norm(hidden_states), past_key_values=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) hidden_states = hidden_states + residual @@ -440,7 +432,6 @@ def forward( past_key_values: Lfm2HybridConvCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -455,20 +446,15 @@ def forward( config=self.config, max_batch_size=batch_size, dtype=self.dtype, device=self.device ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -487,7 +473,6 @@ def forward( position_embeddings=position_embeddings, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/lfm2_moe/modular_lfm2_moe.py b/src/transformers/models/lfm2_moe/modular_lfm2_moe.py index a7d9bf173a11..e5847937ce35 100644 --- a/src/transformers/models/lfm2_moe/modular_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modular_lfm2_moe.py @@ -162,7 +162,6 @@ def forward( past_key_values: Lfm2MoeHybridConvCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -177,20 +176,15 @@ def forward( config=self.config, max_batch_size=batch_size, dtype=self.dtype, device=self.device ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -208,7 +202,6 @@ def forward( attention_mask=layer_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/longcat_flash/modular_longcat_flash.py b/src/transformers/models/longcat_flash/modular_longcat_flash.py index 08c35591a78f..c36b290a4740 100644 --- a/src/transformers/models/longcat_flash/modular_longcat_flash.py +++ b/src/transformers/models/longcat_flash/modular_longcat_flash.py @@ -177,7 +177,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -210,9 +209,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) @@ -273,7 +270,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.Tensor: @@ -286,7 +282,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -309,7 +304,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -378,7 +372,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ): @@ -391,20 +384,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -418,7 +406,6 @@ def forward( attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/ministral/modular_ministral.py b/src/transformers/models/ministral/modular_ministral.py index 736aad12ea00..cdc826b2c5e1 100644 --- a/src/transformers/models/ministral/modular_ministral.py +++ b/src/transformers/models/ministral/modular_ministral.py @@ -210,7 +210,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -222,14 +221,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -238,7 +233,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -258,7 +252,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/ministral3/modular_ministral3.py b/src/transformers/models/ministral3/modular_ministral3.py index a12e131d0226..bde7829d769a 100644 --- a/src/transformers/models/ministral3/modular_ministral3.py +++ b/src/transformers/models/ministral3/modular_ministral3.py @@ -38,7 +38,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -50,6 +49,8 @@ def forward( cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange(query_states.shape[2], device=query_states.device) + past_seen_tokens query_states = query_states * _get_llama_4_attn_scale( cache_position, self.config.rope_parameters.get("llama_4_scaling_beta"), @@ -57,9 +58,7 @@ def forward( ).to(query_states.dtype) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/mistral/modular_mistral.py b/src/transformers/models/mistral/modular_mistral.py index 3708473a2600..c4e8061a77c1 100644 --- a/src/transformers/models/mistral/modular_mistral.py +++ b/src/transformers/models/mistral/modular_mistral.py @@ -56,7 +56,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -70,9 +69,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -121,7 +118,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -133,21 +129,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -162,7 +153,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/moonshine/modular_moonshine.py b/src/transformers/models/moonshine/modular_moonshine.py index fe237200ffc1..af19b293475b 100644 --- a/src/transformers/models/moonshine/modular_moonshine.py +++ b/src/transformers/models/moonshine/modular_moonshine.py @@ -287,7 +287,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, key_value_states: torch.Tensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: @@ -324,19 +323,14 @@ def forward( .transpose(1, 2) ) if is_cross_attention and past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if not is_cross_attention: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, cache_kwargs - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -421,7 +415,6 @@ def forward( encoder_position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, encoder_position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], @@ -435,7 +428,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -610,7 +602,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, encoder_hidden_states: torch.FloatTensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -634,20 +625,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config)) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -670,7 +656,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -700,7 +685,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqModelOutput: r""" @@ -745,7 +729,6 @@ def forward( inputs_embeds=decoder_inputs_embeds, position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -799,7 +782,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, labels: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqLMOutput: @@ -853,7 +835,6 @@ def forward( decoder_inputs_embeds=decoder_inputs_embeds, decoder_position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) logits = self.proj_out(outputs.last_hidden_state) diff --git a/src/transformers/models/moonshine_streaming/modular_moonshine_streaming.py b/src/transformers/models/moonshine_streaming/modular_moonshine_streaming.py index 2f1f6058316a..7553563af288 100644 --- a/src/transformers/models/moonshine_streaming/modular_moonshine_streaming.py +++ b/src/transformers/models/moonshine_streaming/modular_moonshine_streaming.py @@ -359,7 +359,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, encoder_hidden_states: torch.FloatTensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -387,7 +386,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask, **kwargs, diff --git a/src/transformers/models/nanochat/modular_nanochat.py b/src/transformers/models/nanochat/modular_nanochat.py index 617370a5c13f..713cc29b81eb 100644 --- a/src/transformers/models/nanochat/modular_nanochat.py +++ b/src/transformers/models/nanochat/modular_nanochat.py @@ -70,7 +70,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -88,9 +87,7 @@ def forward( key_states = self.k_norm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -153,7 +150,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -166,20 +162,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -195,7 +186,6 @@ def forward( position_embeddings=position_embeddings, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmo/modular_olmo.py b/src/transformers/models/olmo/modular_olmo.py index 8a1294b32a73..6d1588b03e2e 100644 --- a/src/transformers/models/olmo/modular_olmo.py +++ b/src/transformers/models/olmo/modular_olmo.py @@ -117,7 +117,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -140,9 +139,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/olmo2/modular_olmo2.py b/src/transformers/models/olmo2/modular_olmo2.py index 526abae3d6e9..ae0501f193d0 100644 --- a/src/transformers/models/olmo2/modular_olmo2.py +++ b/src/transformers/models/olmo2/modular_olmo2.py @@ -216,7 +216,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -234,9 +233,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -276,7 +273,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -287,7 +283,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/olmo3/modular_olmo3.py b/src/transformers/models/olmo3/modular_olmo3.py index f1a930ee83d0..456a0899890d 100644 --- a/src/transformers/models/olmo3/modular_olmo3.py +++ b/src/transformers/models/olmo3/modular_olmo3.py @@ -212,7 +212,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -230,9 +229,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -286,7 +283,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -299,14 +295,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -315,7 +307,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -334,7 +325,6 @@ def forward( attention_mask=causal_mask_mapping[decoder_layer.self_attn.attention_type], position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/olmoe/modular_olmoe.py b/src/transformers/models/olmoe/modular_olmoe.py index e759ea1b053b..9fee40493496 100644 --- a/src/transformers/models/olmoe/modular_olmoe.py +++ b/src/transformers/models/olmoe/modular_olmoe.py @@ -69,7 +69,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -91,9 +90,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -196,7 +193,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -208,19 +204,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( # diff with mixtral: no sliding config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -238,7 +230,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index b6438baa7729..199ec7ef7b65 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -812,7 +812,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -825,14 +824,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -846,7 +841,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -862,7 +856,6 @@ def forward( position_ids=text_position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1214,7 +1207,6 @@ def forward( pixel_values: torch.Tensor | None = None, image_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | PaddleOCRVLModelOutputWithPast: r""" @@ -1248,7 +1240,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1288,7 +1279,6 @@ def forward( pixel_values: torch.Tensor | None = None, image_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | PaddleOCRVLCausalLMOutputWithPast: @@ -1348,7 +1338,6 @@ def forward( use_cache=use_cache, pixel_values=pixel_values, rope_deltas=rope_deltas, - cache_position=cache_position, **kwargs, ) hidden_states = outputs.last_hidden_state diff --git a/src/transformers/models/phi/modular_phi.py b/src/transformers/models/phi/modular_phi.py index 7c3a33f00243..0ddcdb5ef07a 100644 --- a/src/transformers/models/phi/modular_phi.py +++ b/src/transformers/models/phi/modular_phi.py @@ -91,7 +91,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -123,9 +122,7 @@ def forward( key_states = torch.cat((key_rot, key_pass), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -167,7 +164,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -183,7 +179,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -219,7 +214,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -243,20 +237,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -280,7 +269,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/phi3/modular_phi3.py b/src/transformers/models/phi3/modular_phi3.py index 7cb5c1730034..4229981cc0a8 100644 --- a/src/transformers/models/phi3/modular_phi3.py +++ b/src/transformers/models/phi3/modular_phi3.py @@ -121,7 +121,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -141,9 +140,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -182,7 +179,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -195,7 +191,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -219,7 +214,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -235,7 +229,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = cache_position[0] + past_length = past_key_values.get_seq_length() if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -245,7 +239,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/phimoe/modular_phimoe.py b/src/transformers/models/phimoe/modular_phimoe.py index b24aedf54c73..4ab1b447a03a 100644 --- a/src/transformers/models/phimoe/modular_phimoe.py +++ b/src/transformers/models/phimoe/modular_phimoe.py @@ -367,7 +367,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -383,7 +382,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = cache_position[0] + past_length = past_key_values.get_seq_length() if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -392,7 +391,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/qwen2/modular_qwen2.py b/src/transformers/models/qwen2/modular_qwen2.py index 841f901799d7..5a5e6c0d3455 100644 --- a/src/transformers/models/qwen2/modular_qwen2.py +++ b/src/transformers/models/qwen2/modular_qwen2.py @@ -62,7 +62,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -76,9 +75,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -127,7 +124,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -139,14 +135,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -155,7 +147,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -178,7 +169,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py index 473ca3fdd0a5..57ce864281ea 100644 --- a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py @@ -175,7 +175,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -187,14 +186,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -203,7 +198,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -223,7 +217,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3/modular_qwen3.py b/src/transformers/models/qwen3/modular_qwen3.py index cd5727b5c322..73cde6d89a7a 100644 --- a/src/transformers/models/qwen3/modular_qwen3.py +++ b/src/transformers/models/qwen3/modular_qwen3.py @@ -71,7 +71,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -85,9 +84,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/qwen3_5/modular_qwen3_5.py b/src/transformers/models/qwen3_5/modular_qwen3_5.py index 2c3297386b91..87a7b6f91d12 100644 --- a/src/transformers/models/qwen3_5/modular_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modular_qwen3_5.py @@ -342,7 +342,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: Qwen3_5DynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) @@ -350,12 +349,7 @@ def forward( # Set up dimensions for reshapes later batch_size, seq_len, _ = hidden_states.shape - use_precomputed_states = ( - cache_params is not None - and cache_params.has_previous_state - and seq_len == 1 - and cache_position is not None - ) + use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1 # getting projected states from cache if it exists if cache_params is not None: @@ -490,7 +484,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -502,7 +495,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) elif self.layer_type == "full_attention": @@ -512,7 +504,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -628,7 +619,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -640,15 +630,11 @@ def forward( if use_cache and past_key_values is None: past_key_values = Qwen3_5DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # mrope: the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -662,11 +648,10 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids) @@ -681,7 +666,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -731,7 +715,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3_5ModelOutputWithPast: r""" @@ -784,7 +767,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_moe/modular_qwen3_moe.py b/src/transformers/models/qwen3_moe/modular_qwen3_moe.py index 4dfe23ebfd8a..cf8741aafe2d 100644 --- a/src/transformers/models/qwen3_moe/modular_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modular_qwen3_moe.py @@ -110,7 +110,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -150,7 +149,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_next/modular_qwen3_next.py b/src/transformers/models/qwen3_next/modular_qwen3_next.py index cf6904e72ac6..32d5511581e3 100644 --- a/src/transformers/models/qwen3_next/modular_qwen3_next.py +++ b/src/transformers/models/qwen3_next/modular_qwen3_next.py @@ -164,14 +164,13 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length(layer_idx) kv_length = query_length + past_seen_tokens return kv_length, kv_offset @@ -234,7 +233,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -253,9 +251,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -525,7 +521,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: Qwen3NextDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) @@ -533,12 +528,7 @@ def forward( # Set up dimensions for reshapes later batch_size, seq_len, _ = hidden_states.shape - use_precomputed_states = ( - cache_params is not None - and cache_params.has_previous_state - and seq_len == 1 - and cache_position is not None - ) + use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1 # getting projected states from cache if it exists if cache_params is not None: @@ -684,7 +674,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -696,7 +685,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) elif self.layer_type == "full_attention": @@ -706,7 +694,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -781,7 +768,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -793,23 +779,19 @@ def forward( if use_cache and past_key_values is None: past_key_values = Qwen3NextDynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids) @@ -824,7 +806,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -835,7 +816,7 @@ def forward( past_key_values=past_key_values, ) - def _update_linear_attn_mask(self, attention_mask, cache_position): + def _update_linear_attn_mask(self, attention_mask, past_key_values): """ NOTE: Left-padding is used for linear attention mask. No need for zeroing states when @@ -843,7 +824,9 @@ def _update_linear_attn_mask(self, attention_mask, cache_position): 2. Attending to all inputs """ linear_attn_mask = attention_mask - if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( + attention_mask is not None and torch.all(attention_mask == 1) + ): linear_attn_mask = None return linear_attn_mask @@ -863,7 +846,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -898,7 +880,6 @@ def forward( labels=labels, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) diff --git a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py index 1a964b0701d9..138c9e779ba9 100644 --- a/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modular_qwen3_omni_moe.py @@ -1546,7 +1546,6 @@ def forward( use_cache=None, output_router_logits: bool | None = None, use_audio_in_video=None, - cache_position=None, video_second_per_grid=None, **kwargs, ) -> tuple | Qwen3OmniMoeThinkerCausalLMOutputWithPast: @@ -1654,7 +1653,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, deepstack_visual_embeds=visual_embeds_multiscale, visual_pos_masks=visual_pos_masks, **kwargs, @@ -1761,7 +1759,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if input_ids is not None: @@ -1770,14 +1767,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -1786,7 +1779,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -1807,7 +1799,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1846,7 +1837,6 @@ def forward( inputs_embeds=None, labels=None, use_cache=None, - cache_position=None, generation_steps=None, **kwargs, ): @@ -1870,7 +1860,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -2036,7 +2025,6 @@ def forward( labels=None, use_cache=None, output_router_logits=None, - cache_position=None, residual_codes=None, trailing_text_hidden=None, tts_pad_embed=None, @@ -2101,7 +2089,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) @@ -2149,7 +2136,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, is_first_iteration=False, **kwargs, ): @@ -2159,7 +2145,6 @@ def prepare_inputs_for_generation( past_key_values, attention_mask, inputs_embeds, - cache_position, is_first_iteration=is_first_iteration, **kwargs, ) @@ -2333,7 +2318,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: """ @@ -2349,8 +2333,6 @@ def forward( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model @@ -2366,7 +2348,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + self.self_attn_layer_scale(hidden_states) @@ -2404,7 +2385,6 @@ def forward( past_key_values=None, inputs_embeds=None, use_cache=None, - cache_position=None, **kwargs, ): if input_ids is not None: @@ -2416,7 +2396,6 @@ def forward( past_key_values, inputs_embeds, use_cache, - cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py index 4f6d1ad60841..8225dee92580 100644 --- a/src/transformers/models/qwen3_vl/modular_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modular_qwen3_vl.py @@ -424,7 +424,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -438,9 +437,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -475,7 +472,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: return super().forward( @@ -485,7 +481,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -753,7 +748,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -777,15 +771,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -799,7 +789,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -816,7 +805,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -994,7 +982,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLModelOutputWithPast: r""" @@ -1076,7 +1063,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, **kwargs, @@ -1117,7 +1103,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLCausalLMOutputWithPast: @@ -1178,7 +1163,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1207,7 +1191,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1224,7 +1207,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py index b64dcb69a827..7c8055aaf919 100644 --- a/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modular_qwen3_vl_moe.py @@ -349,7 +349,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -373,15 +372,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -395,7 +390,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -412,7 +406,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -457,7 +450,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLMoeCausalLMOutputWithPast: @@ -523,7 +515,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/seed_oss/modular_seed_oss.py b/src/transformers/models/seed_oss/modular_seed_oss.py index 305e36549ee0..5f122bbafc67 100644 --- a/src/transformers/models/seed_oss/modular_seed_oss.py +++ b/src/transformers/models/seed_oss/modular_seed_oss.py @@ -100,7 +100,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -114,9 +113,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/smollm3/modular_smollm3.py b/src/transformers/models/smollm3/modular_smollm3.py index b0c7af7c3c22..98c72418fbca 100644 --- a/src/transformers/models/smollm3/modular_smollm3.py +++ b/src/transformers/models/smollm3/modular_smollm3.py @@ -241,7 +241,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -256,8 +255,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/starcoder2/modular_starcoder2.py b/src/transformers/models/starcoder2/modular_starcoder2.py index 6d0f2b616691..2855e98c5ecd 100644 --- a/src/transformers/models/starcoder2/modular_starcoder2.py +++ b/src/transformers/models/starcoder2/modular_starcoder2.py @@ -81,7 +81,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -95,9 +94,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -152,7 +149,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -164,21 +160,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -196,7 +187,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) From b7922663780730b466f2d833399b1edfa4897f11 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 19:03:05 +0100 Subject: [PATCH 05/42] reapply all those modualrs --- .../models/afmoe/modeling_afmoe.py | 20 +---- .../models/apertus/modeling_apertus.py | 6 +- src/transformers/models/aria/modeling_aria.py | 6 -- .../audioflamingo3/modeling_audioflamingo3.py | 7 +- .../models/bitnet/modeling_bitnet.py | 5 +- .../models/cohere/modeling_cohere.py | 11 +-- .../models/cohere2/modeling_cohere2.py | 4 - src/transformers/models/cwm/modeling_cwm.py | 18 +---- .../deepseek_v2/modeling_deepseek_v2.py | 5 +- .../deepseek_v3/modeling_deepseek_v3.py | 5 +- src/transformers/models/dia/modeling_dia.py | 17 +--- .../models/diffllama/modeling_diffllama.py | 15 +--- .../models/dots1/modeling_dots1.py | 18 +---- src/transformers/models/emu3/modeling_emu3.py | 8 -- .../modeling_ernie4_5_vl_moe.py | 23 +----- .../models/evolla/modeling_evolla.py | 15 +--- .../models/exaone4/modeling_exaone4.py | 22 +----- .../models/exaone_moe/modeling_exaone_moe.py | 20 +---- .../models/flex_olmo/modeling_flex_olmo.py | 5 +- .../models/gemma/modeling_gemma.py | 13 +-- src/transformers/models/glm4/modeling_glm4.py | 2 - .../models/glm4_moe/modeling_glm4_moe.py | 5 +- .../glm4_moe_lite/modeling_glm4_moe_lite.py | 5 +- .../models/glm4v_moe/modeling_glm4v_moe.py | 26 ++++-- .../glm_moe_dsa/modeling_glm_moe_dsa.py | 4 +- .../models/glmasr/modeling_glmasr.py | 7 +- .../models/granite/modeling_granite.py | 19 +---- .../models/granitemoe/modeling_granitemoe.py | 16 +--- .../modeling_granitemoehybrid.py | 2 - .../modeling_granitemoeshared.py | 16 +--- .../modeling_hunyuan_v1_dense.py | 5 +- .../hunyuan_v1_moe/modeling_hunyuan_v1_moe.py | 5 +- src/transformers/models/lfm2/modeling_lfm2.py | 35 +++----- .../models/lfm2_moe/modeling_lfm2_moe.py | 35 +++----- .../longcat_flash/modeling_longcat_flash.py | 21 +---- .../models/minimax/modeling_minimax.py | 5 +- .../models/minimax_m2/modeling_minimax_m2.py | 5 +- .../models/ministral/modeling_ministral.py | 18 +---- .../models/ministral3/modeling_ministral3.py | 20 ++--- .../models/mistral/modeling_mistral.py | 18 +---- .../models/mixtral/modeling_mixtral.py | 5 +- .../models/moonshine/modeling_moonshine.py | 29 ++----- .../modeling_moonshine_streaming.py | 29 ++----- .../models/nanochat/modeling_nanochat.py | 18 +---- src/transformers/models/olmo/modeling_olmo.py | 5 +- .../models/olmo2/modeling_olmo2.py | 7 +- .../models/olmo3/modeling_olmo3.py | 20 +---- .../models/olmoe/modeling_olmoe.py | 17 +--- .../paddleocr_vl/modeling_paddleocr_vl.py | 17 +--- src/transformers/models/phi/modeling_phi.py | 20 +---- src/transformers/models/phi3/modeling_phi3.py | 24 ++---- .../modeling_phi4_multimodal.py | 7 +- .../models/phimoe/modular_phimoe.py | 4 +- .../models/qwen2/modeling_qwen2.py | 18 +---- .../models/qwen2_moe/modeling_qwen2_moe.py | 13 +-- .../models/qwen3/modeling_qwen3.py | 18 +---- .../models/qwen3_5/modeling_qwen3_5.py | 46 +++-------- .../qwen3_5_moe/modeling_qwen3_5_moe.py | 47 +++-------- .../models/qwen3_moe/modeling_qwen3_moe.py | 7 +- .../models/qwen3_next/modeling_qwen3_next.py | 40 +++------- .../qwen3_omni_moe/modeling_qwen3_omni_moe.py | 79 ++++--------------- .../models/qwen3_vl/modeling_qwen3_vl.py | 25 +----- .../qwen3_vl_moe/modeling_qwen3_vl_moe.py | 24 +----- .../models/seed_oss/modeling_seed_oss.py | 5 +- .../models/smollm3/modeling_smollm3.py | 17 +--- .../models/starcoder2/modeling_starcoder2.py | 18 +---- .../modeling_voxtral_realtime.py | 25 ++---- .../models/youtu/modeling_youtu.py | 5 +- 68 files changed, 245 insertions(+), 856 deletions(-) diff --git a/src/transformers/models/afmoe/modeling_afmoe.py b/src/transformers/models/afmoe/modeling_afmoe.py index fe621838838e..7fb24d347c00 100644 --- a/src/transformers/models/afmoe/modeling_afmoe.py +++ b/src/transformers/models/afmoe/modeling_afmoe.py @@ -382,7 +382,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_value: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -402,8 +401,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_value is not None: - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -465,7 +463,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_value: Cache | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: @@ -479,7 +476,6 @@ def forward( position_ids=position_ids, past_key_value=past_key_value, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -569,7 +565,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: @@ -582,15 +577,10 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, - past_seen_tokens + inputs_embeds.shape[1], - device=inputs_embeds.device, - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -598,7 +588,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, } causal_mask_mapping = { @@ -621,7 +610,6 @@ def forward( position_ids=position_ids, past_key_value=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/apertus/modeling_apertus.py b/src/transformers/models/apertus/modeling_apertus.py index dc2d7329a186..213dcb97082d 100644 --- a/src/transformers/models/apertus/modeling_apertus.py +++ b/src/transformers/models/apertus/modeling_apertus.py @@ -247,7 +247,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -263,8 +262,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -304,7 +302,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor]: @@ -316,7 +313,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 07d107a13d8b..51ec79c982ec 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -969,7 +969,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple | AriaModelOutputWithPast: if inputs_embeds is None: @@ -995,7 +994,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1085,7 +1083,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, logits_to_keep: int | torch.Tensor = 0, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | AriaCausalLMOutputWithPast: r""" @@ -1158,7 +1155,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1189,7 +1185,6 @@ def prepare_inputs_for_generation( pixel_values=None, pixel_mask=None, attention_mask=None, - cache_position=None, logits_to_keep=None, is_first_iteration=False, **kwargs, @@ -1199,7 +1194,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, logits_to_keep=logits_to_keep, is_first_iteration=is_first_iteration, **kwargs, diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index f635206721db..270ededa0230 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -496,7 +496,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -582,22 +581,20 @@ def forward( past_key_values=past_key_values, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) return outputs - def prepare_inputs_for_generation(self, *args, **kwargs): + def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, **kwargs): # Overwritten -- we should not pass input_features when we are in cached decoding stage input_features = kwargs.pop("input_features", None) input_features_mask = kwargs.pop("input_features_mask", None) - cache_position = kwargs.get("cache_position") model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if cache_position is not None and model_inputs["cache_position"][0] == 0: + if is_first_iteration: # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features diff --git a/src/transformers/models/bitnet/modeling_bitnet.py b/src/transformers/models/bitnet/modeling_bitnet.py index 57eda03e5d57..14c1581b250f 100644 --- a/src/transformers/models/bitnet/modeling_bitnet.py +++ b/src/transformers/models/bitnet/modeling_bitnet.py @@ -182,7 +182,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -196,9 +195,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/cohere/modeling_cohere.py b/src/transformers/models/cohere/modeling_cohere.py index 4128072a0c1c..d85b890d829c 100644 --- a/src/transformers/models/cohere/modeling_cohere.py +++ b/src/transformers/models/cohere/modeling_cohere.py @@ -262,7 +262,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -284,9 +283,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; position_ids needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -323,7 +320,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -340,8 +336,6 @@ def forward( use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. @@ -355,7 +349,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -488,7 +481,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -529,7 +521,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/cohere2/modeling_cohere2.py b/src/transformers/models/cohere2/modeling_cohere2.py index 186f5a323046..ac0aeba78f3b 100644 --- a/src/transformers/models/cohere2/modeling_cohere2.py +++ b/src/transformers/models/cohere2/modeling_cohere2.py @@ -317,8 +317,6 @@ def forward( use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. @@ -474,7 +472,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -515,7 +512,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/cwm/modeling_cwm.py b/src/transformers/models/cwm/modeling_cwm.py index dc16b3ddea5f..772076754eb2 100644 --- a/src/transformers/models/cwm/modeling_cwm.py +++ b/src/transformers/models/cwm/modeling_cwm.py @@ -203,7 +203,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -217,9 +216,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -375,7 +372,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> CwmModelOutputWithPast: @@ -388,21 +384,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) if not isinstance(causal_mask_mapping := attention_mask, dict): mask_kwargs = { "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -422,7 +413,6 @@ def forward( attention_mask=causal_mask_mapping[decoder_layer.attention_type], position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py index e6618868c722..b57894c04d7e 100644 --- a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py +++ b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py @@ -339,7 +339,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: @@ -367,9 +366,7 @@ def forward( key_states = torch.cat((k_nope, k_pe), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index 3fac50a92e24..ab998cc99c21 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -419,7 +419,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -452,9 +451,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) diff --git a/src/transformers/models/dia/modeling_dia.py b/src/transformers/models/dia/modeling_dia.py index 8bd11629e675..12cb3a72524b 100644 --- a/src/transformers/models/dia/modeling_dia.py +++ b/src/transformers/models/dia/modeling_dia.py @@ -526,7 +526,6 @@ def forward( encoder_hidden_states: torch.Tensor | None = None, encoder_attention_mask: torch.Tensor | None = None, past_key_values: EncoderDecoderCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: self_attn_cache = past_key_values @@ -542,7 +541,6 @@ def forward( # Needs to be an arg in order to function properly # on inplace operations to be carried (e.g. compile) self_attn_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + self_attn_output @@ -594,7 +592,6 @@ def forward( past_key_values: EncoderDecoderCache | None = None, output_attentions: bool | None = False, output_hidden_states: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> BaseModelOutputWithPastAndCrossAttentions | tuple: r""" @@ -606,12 +603,10 @@ def forward( batch_size, seq_length = input_ids.size()[:-1] past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 0 - if cache_position is None: - cache_position = torch.arange( - past_key_values_length, past_key_values_length + seq_length, device=input_ids.device - ) + if position_ids is None: - position_ids = cache_position[None, :] + position_ids = torch.arange(seq_length, device=input_ids.device) + past_key_values_length + position_ids = position_ids.unsqueeze(0) # RoPE hidden_states = self.embeddings(input_ids) @@ -625,7 +620,6 @@ def forward( config=self.config, inputs_embeds=hidden_states, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, ) encoder_attention_mask = create_bidirectional_mask( @@ -653,7 +647,6 @@ def forward( encoder_hidden_states, encoder_attention_mask=encoder_attention_mask, past_key_values=past_key_values, - cache_position=cache_position, position_ids=position_ids, **kwargs, ) @@ -706,7 +699,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | Seq2SeqModelOutput: r""" @@ -788,7 +780,6 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -843,7 +834,6 @@ def forward( output_attentions: bool | None = None, output_hidden_states: bool | None = None, labels: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | Seq2SeqLMOutput: r""" @@ -883,7 +873,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index c03fcd902e2f..d80ccd572dc3 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -227,7 +227,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: bsz, target_len, _ = hidden_states.size() @@ -245,9 +244,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) @@ -304,7 +301,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, ) -> tuple[torch.Tensor, None]: if isinstance(past_key_values, StaticCache): raise ValueError( @@ -329,9 +325,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache # to be able to avoid many of these transpose/reshape/view. @@ -432,7 +426,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: bsz, q_len, _ = hidden_states.size() @@ -449,9 +442,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) key_states = repeat_kv(key_states, self.num_key_value_groups) value_states = repeat_kv(value_states, self.num_key_value_groups) diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index 998fc7004b22..998af195ecd5 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -240,7 +240,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -254,9 +253,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -515,7 +512,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -527,14 +523,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -543,7 +535,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -566,7 +557,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index deca1ef911d1..34d22687c9a7 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -251,7 +251,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -264,7 +263,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1471,7 +1469,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | CausalLMOutputWithPast: r""" @@ -1503,7 +1500,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1550,7 +1546,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, labels: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], @@ -1610,7 +1605,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1639,7 +1633,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1653,7 +1646,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, use_cache=use_cache, diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index caa82d76c74d..dae8626818a2 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -512,7 +512,6 @@ def forward( position_ids: torch.Tensor | None = None, moe_mm_token_type_ids: torch.IntTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: residual = hidden_states @@ -526,7 +525,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = hidden_states + residual @@ -744,7 +742,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> MoeModelOutputWithPast: r""" @@ -760,15 +757,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -789,7 +782,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -807,7 +799,6 @@ def forward( position_ids=position_ids, moe_mm_token_type_ids=moe_mm_token_type_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) @@ -1353,7 +1344,7 @@ def compute_3d_position_ids( delta = self.rope_deltas.repeat_interleave(batch_size // self.rope_deltas.shape[0], dim=0) position_ids = (position_ids + delta).to(device=inputs_embeds.device) else: - # Can't build correct 3D positions. Let the model infer it from `cache_position` + # Can't build correct 3D positions. Let the model infer it position_ids = None return position_ids @@ -1374,7 +1365,6 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: r""" @@ -1428,7 +1418,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, return_dict=True, - cache_position=cache_position, **kwargs, ) @@ -1597,7 +1586,6 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeCausalLMOutputWithPast: @@ -1637,7 +1625,6 @@ def forward( image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, rope_deltas=rope_deltas, - cache_position=cache_position, **kwargs, ) @@ -1676,7 +1663,6 @@ def prepare_inputs_for_generation( input_ids, inputs_embeds=None, attention_mask=None, - cache_position=None, past_key_values=None, image_grid_thw=None, video_grid_thw=None, @@ -1689,7 +1675,6 @@ def prepare_inputs_for_generation( input_ids, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw, diff --git a/src/transformers/models/evolla/modeling_evolla.py b/src/transformers/models/evolla/modeling_evolla.py index 238da9b2a0ff..50569732061d 100644 --- a/src/transformers/models/evolla/modeling_evolla.py +++ b/src/transformers/models/evolla/modeling_evolla.py @@ -1189,7 +1189,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, protein_kv_states: torch.Tensor | None = None, structure_kv_states: torch.Tensor | None = None, msa_kv_states: torch.Tensor | None = None, @@ -1210,7 +1209,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1310,7 +1308,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, protein_input_ids: torch.LongTensor | None = None, protein_attention_mask: torch.Tensor | None = None, structure_feats: torch.FloatTensor | None = None, @@ -1342,14 +1339,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) protein_feats = None protein_batch_mask = None @@ -1370,7 +1363,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, ) @@ -1384,7 +1376,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, protein_kv_states=protein_feats, structure_kv_states=structure_feats, msa_kv_states=msa_feats, diff --git a/src/transformers/models/exaone4/modeling_exaone4.py b/src/transformers/models/exaone4/modeling_exaone4.py index a623e0f8d324..fab10b9b6937 100644 --- a/src/transformers/models/exaone4/modeling_exaone4.py +++ b/src/transformers/models/exaone4/modeling_exaone4.py @@ -234,7 +234,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -254,10 +253,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = { - "cache_position": cache_position, - } - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -313,7 +309,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -324,7 +319,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -387,7 +381,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -399,14 +392,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -415,7 +404,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -437,7 +425,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -476,7 +463,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: diff --git a/src/transformers/models/exaone_moe/modeling_exaone_moe.py b/src/transformers/models/exaone_moe/modeling_exaone_moe.py index 2f684c18e618..2836a3c2245d 100644 --- a/src/transformers/models/exaone_moe/modeling_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modeling_exaone_moe.py @@ -166,7 +166,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -186,10 +185,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = { - "cache_position": cache_position, - } - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -508,7 +504,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -520,14 +515,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -536,7 +527,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -558,7 +548,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -597,7 +586,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: diff --git a/src/transformers/models/flex_olmo/modeling_flex_olmo.py b/src/transformers/models/flex_olmo/modeling_flex_olmo.py index aa6ce9f5723a..aa771fa55ce4 100644 --- a/src/transformers/models/flex_olmo/modeling_flex_olmo.py +++ b/src/transformers/models/flex_olmo/modeling_flex_olmo.py @@ -249,7 +249,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -267,9 +266,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py index 5f95e5612c77..da3f4a03774c 100644 --- a/src/transformers/models/gemma/modeling_gemma.py +++ b/src/transformers/models/gemma/modeling_gemma.py @@ -383,7 +383,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -395,20 +394,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -430,7 +424,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/glm4/modeling_glm4.py b/src/transformers/models/glm4/modeling_glm4.py index a9b00476e5e4..e99930ae57f6 100644 --- a/src/transformers/models/glm4/modeling_glm4.py +++ b/src/transformers/models/glm4/modeling_glm4.py @@ -83,7 +83,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -96,7 +95,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/glm4_moe/modeling_glm4_moe.py b/src/transformers/models/glm4_moe/modeling_glm4_moe.py index 6b6c77ff4c72..1bc20c8322d9 100644 --- a/src/transformers/models/glm4_moe/modeling_glm4_moe.py +++ b/src/transformers/models/glm4_moe/modeling_glm4_moe.py @@ -226,7 +226,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -248,9 +247,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; position_ids needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py index ed8cf6a478d8..d59fd2ab996e 100644 --- a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py +++ b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py @@ -284,7 +284,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -317,9 +316,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 32678a2dcd4d..2474100bc8d4 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -157,6 +157,7 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, + cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -174,7 +175,9 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + # sin and cos are specific to RoPE models; position_ids needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -990,6 +993,7 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, + cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -1002,11 +1006,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + # the hard coded `3` is for temporal, height and width. if position_ids is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -1031,6 +1039,7 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, + "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": text_position_ids, } @@ -1049,6 +1058,7 @@ def forward( attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, + cache_position=cache_position, **kwargs, ) hidden_states = layer_outputs @@ -1386,7 +1396,7 @@ def compute_3d_position_ids( delta = self.rope_deltas.repeat_interleave(batch_size // self.rope_deltas.shape[0], dim=0) position_ids = position_ids + delta.to(device=position_ids.device) else: - # Can't build correct 3D positions. Let the model infer it + # Can't build correct 3D positions. Let the model infer it from `cache_position` position_ids = None return position_ids @@ -1404,6 +1414,7 @@ def forward( image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, + cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Glm4vMoeModelOutputWithPast: r""" @@ -1448,6 +1459,7 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, + cache_position=cache_position, **kwargs, ) @@ -1606,6 +1618,7 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, + cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Glm4vMoeCausalLMOutputWithPast: @@ -1661,6 +1674,7 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, + cache_position=cache_position, **kwargs, ) @@ -1703,6 +1717,7 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, + cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1719,6 +1734,7 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, + cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py index ab0286129710..9c8017582e47 100644 --- a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py @@ -342,7 +342,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -383,8 +382,7 @@ def forward( # Cache update if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # ===== Indexer (DSA sparse mask) ===== # attention_mask is [B, 1, S, T] (4D) for eager and (2D) otherwise but indexer works with [B, S, T] (3D) diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 09f65935d8b5..271f4f8c8bfb 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -438,7 +438,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -490,22 +489,20 @@ def forward( past_key_values=past_key_values, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, **kwargs, ) return outputs - def prepare_inputs_for_generation(self, *args, **kwargs): + def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, **kwargs): # Overwritten -- we should not pass input_features when we are in cached decoding stage input_features = kwargs.pop("input_features", None) input_features_mask = kwargs.pop("input_features_mask", None) - cache_position = kwargs.get("cache_position") model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if cache_position is not None and model_inputs["cache_position"][0] == 0: + if is_first_iteration: # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features diff --git a/src/transformers/models/granite/modeling_granite.py b/src/transformers/models/granite/modeling_granite.py index 678b4df53a2f..5723ccadeecb 100644 --- a/src/transformers/models/granite/modeling_granite.py +++ b/src/transformers/models/granite/modeling_granite.py @@ -238,7 +238,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -255,8 +254,6 @@ def forward( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Cache`, *optional*): cached past key and value projection states - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. @@ -276,7 +273,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -412,7 +408,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -438,20 +433,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -474,7 +464,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -526,7 +515,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -562,7 +550,6 @@ def forward( use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoe/modeling_granitemoe.py b/src/transformers/models/granitemoe/modeling_granitemoe.py index aea9b676c95c..5fb53d6afe49 100644 --- a/src/transformers/models/granitemoe/modeling_granitemoe.py +++ b/src/transformers/models/granitemoe/modeling_granitemoe.py @@ -419,7 +419,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> torch.Tensor: @@ -429,7 +428,6 @@ def forward( hidden_states=hidden_states, attention_mask=attention_mask, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -495,7 +493,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -507,19 +504,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( # ONLY DIFF WITH MIXTRAL: NO SLIDING config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -537,7 +530,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -661,7 +653,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | MoeCausalLMOutputWithPast: @@ -697,7 +688,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py index 2e1625742cce..3b25c2d977b9 100644 --- a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py @@ -1478,7 +1478,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | MoeCausalLMOutputWithPast: @@ -1514,7 +1513,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py index 8a3536df212a..71f8c6eaff7d 100644 --- a/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py +++ b/src/transformers/models/granitemoeshared/modeling_granitemoeshared.py @@ -411,7 +411,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[GraniteFlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -426,7 +425,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -564,7 +562,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -576,19 +573,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( # ONLY DIFF WITH MIXTRAL: NO SLIDING config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -606,7 +599,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -730,7 +722,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | MoeCausalLMOutputWithPast: @@ -766,7 +757,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py index 9be66c29776a..d1652d78cbbc 100644 --- a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py +++ b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py @@ -186,7 +186,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -202,9 +201,7 @@ def forward( key_states = self.key_layernorm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py index a7043296afd1..19779da0528c 100644 --- a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py +++ b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py @@ -189,7 +189,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -205,9 +204,7 @@ def forward( key_states = self.key_layernorm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/lfm2/modeling_lfm2.py b/src/transformers/models/lfm2/modeling_lfm2.py index db24d85ad0ca..c42eb81064ef 100644 --- a/src/transformers/models/lfm2/modeling_lfm2.py +++ b/src/transformers/models/lfm2/modeling_lfm2.py @@ -251,7 +251,7 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. @@ -259,7 +259,6 @@ def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[ for each layer. """ full_mask_kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length() kv_length = query_length + past_seen_tokens return kv_length, full_mask_kv_offset @@ -381,7 +380,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -395,8 +393,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -460,7 +457,6 @@ def cuda_kernels_forward( self, x: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): x = apply_mask_to_padding_states(x, attention_mask) @@ -470,7 +466,7 @@ def cuda_kernels_forward( Bx = B * x conv_weights = self.conv.weight.view(self.conv.weight.size(0), self.conv.weight.size(2)) - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_out = causal_conv1d_update( Bx.squeeze(-1), past_key_values.conv_cache[self.layer_idx], @@ -494,7 +490,6 @@ def slow_forward( self, x: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): seqlen = x.shape[1] @@ -505,8 +500,9 @@ def slow_forward( Bx = B * x - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] + cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) @@ -532,12 +528,11 @@ def forward( self, hidden_states: torch.Tensor, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): if is_fast_path_available and "cuda" in hidden_states.device.type and not is_torchdynamo_compiling(): - return self.cuda_kernels_forward(hidden_states, past_key_values, cache_position, attention_mask) - return self.slow_forward(hidden_states, past_key_values, cache_position, attention_mask) + return self.cuda_kernels_forward(hidden_states, past_key_values, attention_mask) + return self.slow_forward(hidden_states, past_key_values, attention_mask) class Lfm2DecoderLayer(GradientCheckpointingLayer): @@ -560,7 +555,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Lfm2HybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> torch.Tensor: residual = hidden_states @@ -571,14 +565,12 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) else: hidden_states = self.conv( hidden_states=self.operator_norm(hidden_states), past_key_values=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) hidden_states = hidden_states + residual @@ -634,7 +626,6 @@ def forward( past_key_values: Lfm2HybridConvCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -649,20 +640,15 @@ def forward( config=self.config, max_batch_size=batch_size, dtype=self.dtype, device=self.device ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -681,7 +667,6 @@ def forward( position_embeddings=position_embeddings, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py index a659342bc53a..b0b50ddbcf40 100644 --- a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py @@ -327,7 +327,7 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. @@ -335,7 +335,6 @@ def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[ for each layer. """ full_mask_kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length() kv_length = query_length + past_seen_tokens return kv_length, full_mask_kv_offset @@ -457,7 +456,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Lfm2MoeHybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -471,8 +469,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -536,7 +533,6 @@ def cuda_kernels_forward( self, x: torch.Tensor, past_key_values: Lfm2MoeHybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): x = apply_mask_to_padding_states(x, attention_mask) @@ -546,7 +542,7 @@ def cuda_kernels_forward( Bx = B * x conv_weights = self.conv.weight.view(self.conv.weight.size(0), self.conv.weight.size(2)) - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_out = causal_conv1d_update( Bx.squeeze(-1), past_key_values.conv_cache[self.layer_idx], @@ -570,7 +566,6 @@ def slow_forward( self, x: torch.Tensor, past_key_values: Lfm2MoeHybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): seqlen = x.shape[1] @@ -581,8 +576,9 @@ def slow_forward( Bx = B * x - if past_key_values is not None and cache_position[0] > 0: + if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] + cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) @@ -608,12 +604,11 @@ def forward( self, hidden_states: torch.Tensor, past_key_values: Lfm2MoeHybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): if is_fast_path_available and "cuda" in hidden_states.device.type and not is_torchdynamo_compiling(): - return self.cuda_kernels_forward(hidden_states, past_key_values, cache_position, attention_mask) - return self.slow_forward(hidden_states, past_key_values, cache_position, attention_mask) + return self.cuda_kernels_forward(hidden_states, past_key_values, attention_mask) + return self.slow_forward(hidden_states, past_key_values, attention_mask) class Lfm2MoeDecoderLayer(GradientCheckpointingLayer): @@ -640,7 +635,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Lfm2MoeHybridConvCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> torch.Tensor: residual = hidden_states @@ -651,14 +645,12 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) else: hidden_states = self.conv( hidden_states=self.operator_norm(hidden_states), past_key_values=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) hidden_states = hidden_states + residual @@ -724,7 +716,6 @@ def forward( past_key_values: Lfm2MoeHybridConvCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -739,20 +730,15 @@ def forward( config=self.config, max_batch_size=batch_size, dtype=self.dtype, device=self.device ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -770,7 +756,6 @@ def forward( attention_mask=layer_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/longcat_flash/modeling_longcat_flash.py b/src/transformers/models/longcat_flash/modeling_longcat_flash.py index 4f972fa88e5e..d84cef76c80e 100644 --- a/src/transformers/models/longcat_flash/modeling_longcat_flash.py +++ b/src/transformers/models/longcat_flash/modeling_longcat_flash.py @@ -393,7 +393,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -426,9 +425,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) @@ -489,7 +486,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.Tensor: @@ -502,7 +498,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -525,7 +520,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -602,7 +596,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -615,20 +608,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -642,7 +630,6 @@ def forward( attention_mask=causal_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/minimax/modeling_minimax.py b/src/transformers/models/minimax/modeling_minimax.py index 028068a78a77..eefdced42b66 100644 --- a/src/transformers/models/minimax/modeling_minimax.py +++ b/src/transformers/models/minimax/modeling_minimax.py @@ -418,7 +418,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -432,9 +431,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/minimax_m2/modeling_minimax_m2.py b/src/transformers/models/minimax_m2/modeling_minimax_m2.py index e4e276294324..9abba971208b 100644 --- a/src/transformers/models/minimax_m2/modeling_minimax_m2.py +++ b/src/transformers/models/minimax_m2/modeling_minimax_m2.py @@ -318,7 +318,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -336,9 +335,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/ministral/modeling_ministral.py b/src/transformers/models/ministral/modeling_ministral.py index d66c5ff04371..d9856ca49694 100644 --- a/src/transformers/models/ministral/modeling_ministral.py +++ b/src/transformers/models/ministral/modeling_ministral.py @@ -145,7 +145,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -159,9 +158,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -362,7 +359,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -374,14 +370,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -390,7 +382,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -410,7 +401,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py index 646904a06e74..be0659788d71 100644 --- a/src/transformers/models/ministral3/modeling_ministral3.py +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -131,7 +131,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -143,6 +142,8 @@ def forward( cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange(query_states.shape[2], device=query_states.device) + past_seen_tokens query_states = query_states * _get_llama_4_attn_scale( cache_position, self.config.rope_parameters.get("llama_4_scaling_beta"), @@ -150,9 +151,7 @@ def forward( ).to(query_states.dtype) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -366,7 +365,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -378,21 +376,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -407,7 +400,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py index 1f457ecf0695..b79dea36c9e9 100644 --- a/src/transformers/models/mistral/modeling_mistral.py +++ b/src/transformers/models/mistral/modeling_mistral.py @@ -142,7 +142,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -156,9 +155,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -356,7 +353,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -368,21 +364,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -397,7 +388,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/mixtral/modeling_mixtral.py b/src/transformers/models/mixtral/modeling_mixtral.py index 859300d4bb69..484077d48115 100644 --- a/src/transformers/models/mixtral/modeling_mixtral.py +++ b/src/transformers/models/mixtral/modeling_mixtral.py @@ -315,7 +315,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -329,9 +328,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/moonshine/modeling_moonshine.py b/src/transformers/models/moonshine/modeling_moonshine.py index b48b25096a07..00728219f3f8 100644 --- a/src/transformers/models/moonshine/modeling_moonshine.py +++ b/src/transformers/models/moonshine/modeling_moonshine.py @@ -287,7 +287,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, key_value_states: torch.Tensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: @@ -324,19 +323,14 @@ def forward( .transpose(1, 2) ) if is_cross_attention and past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if not is_cross_attention: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, cache_kwargs - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -453,7 +447,6 @@ def forward( encoder_position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, encoder_position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], @@ -467,7 +460,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -652,7 +644,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, encoder_hidden_states: torch.FloatTensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -676,20 +667,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config)) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -712,7 +698,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -768,7 +753,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqModelOutput: r""" @@ -813,7 +797,6 @@ def forward( inputs_embeds=decoder_inputs_embeds, position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -883,7 +866,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, labels: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqLMOutput: @@ -937,7 +919,6 @@ def forward( decoder_inputs_embeds=decoder_inputs_embeds, decoder_position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) logits = self.proj_out(outputs.last_hidden_state) diff --git a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py index 167adb7f6fcf..ec29e7fc51c3 100644 --- a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py +++ b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py @@ -616,7 +616,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, key_value_states: torch.Tensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: @@ -653,19 +652,14 @@ def forward( .transpose(1, 2) ) if is_cross_attention and past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if not is_cross_attention: cos, sin = position_embeddings query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, cache_kwargs - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -733,7 +727,6 @@ def forward( encoder_position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, encoder_position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], @@ -747,7 +740,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -813,7 +805,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, encoder_hidden_states: torch.FloatTensor | None = None, encoder_attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -842,20 +833,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config)) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -878,7 +864,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -933,7 +918,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqModelOutput: r""" @@ -978,7 +962,6 @@ def forward( inputs_embeds=decoder_inputs_embeds, position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1048,7 +1031,6 @@ def forward( decoder_inputs_embeds: tuple[torch.FloatTensor] | None = None, decoder_position_ids: tuple[torch.LongTensor] | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, labels: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> Seq2SeqLMOutput: @@ -1102,7 +1084,6 @@ def forward( decoder_inputs_embeds=decoder_inputs_embeds, decoder_position_ids=decoder_position_ids, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) logits = self.proj_out(outputs.last_hidden_state) diff --git a/src/transformers/models/nanochat/modeling_nanochat.py b/src/transformers/models/nanochat/modeling_nanochat.py index e680b3cbaddd..23ac56cbf90a 100644 --- a/src/transformers/models/nanochat/modeling_nanochat.py +++ b/src/transformers/models/nanochat/modeling_nanochat.py @@ -228,7 +228,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -246,9 +245,7 @@ def forward( key_states = self.k_norm(key_states) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -386,7 +383,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -399,20 +395,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -428,7 +419,6 @@ def forward( position_embeddings=position_embeddings, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmo/modeling_olmo.py b/src/transformers/models/olmo/modeling_olmo.py index df73ff2b7e6a..a0949e957057 100644 --- a/src/transformers/models/olmo/modeling_olmo.py +++ b/src/transformers/models/olmo/modeling_olmo.py @@ -243,7 +243,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -266,9 +265,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/olmo2/modeling_olmo2.py b/src/transformers/models/olmo2/modeling_olmo2.py index 465af484bf63..01fe9fb158b8 100644 --- a/src/transformers/models/olmo2/modeling_olmo2.py +++ b/src/transformers/models/olmo2/modeling_olmo2.py @@ -237,7 +237,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -255,9 +254,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -312,7 +309,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -323,7 +319,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/olmo3/modeling_olmo3.py b/src/transformers/models/olmo3/modeling_olmo3.py index 23423be782aa..cea28efa101c 100644 --- a/src/transformers/models/olmo3/modeling_olmo3.py +++ b/src/transformers/models/olmo3/modeling_olmo3.py @@ -171,7 +171,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -189,9 +188,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -247,7 +244,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -258,7 +254,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -385,7 +380,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -398,14 +392,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -414,7 +404,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -433,7 +422,6 @@ def forward( attention_mask=causal_mask_mapping[decoder_layer.self_attn.attention_type], position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index 93b17496d111..38d3f6c95411 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -254,7 +254,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -276,9 +275,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -474,7 +471,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -486,19 +482,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( # diff with mixtral: no sliding config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -516,7 +508,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index b0774abea9f0..dd1e6c6a3498 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -482,7 +482,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -495,14 +494,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = ( - torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - ) - if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -516,7 +511,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -532,7 +526,6 @@ def forward( position_ids=text_position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1296,7 +1289,6 @@ def forward( pixel_values: torch.Tensor | None = None, image_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple | PaddleOCRVLModelOutputWithPast: r""" @@ -1330,7 +1322,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1396,7 +1387,6 @@ def forward( pixel_values: torch.Tensor | None = None, image_grid_thw: torch.LongTensor | None = None, rope_deltas: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | PaddleOCRVLCausalLMOutputWithPast: @@ -1456,7 +1446,6 @@ def forward( use_cache=use_cache, pixel_values=pixel_values, rope_deltas=rope_deltas, - cache_position=cache_position, **kwargs, ) hidden_states = outputs.last_hidden_state diff --git a/src/transformers/models/phi/modeling_phi.py b/src/transformers/models/phi/modeling_phi.py index 2f4906a61d29..08fb09da909e 100644 --- a/src/transformers/models/phi/modeling_phi.py +++ b/src/transformers/models/phi/modeling_phi.py @@ -203,7 +203,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -235,9 +234,7 @@ def forward( key_states = torch.cat((key_rot, key_pass), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -290,7 +287,6 @@ def forward( past_key_values: Cache | None = None, output_attentions: bool | None = False, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -306,7 +302,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -373,7 +368,6 @@ def forward( use_cache: bool | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions @@ -397,20 +391,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -434,7 +423,6 @@ def forward( past_key_values=past_key_values, output_attentions=output_attentions, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/phi3/modeling_phi3.py b/src/transformers/models/phi3/modeling_phi3.py index 3de9c5edd3d9..b07735f8a2e6 100644 --- a/src/transformers/models/phi3/modeling_phi3.py +++ b/src/transformers/models/phi3/modeling_phi3.py @@ -229,7 +229,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -249,9 +248,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -314,7 +311,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -327,7 +323,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -389,7 +384,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -401,21 +395,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -430,7 +419,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -520,7 +508,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -536,7 +523,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = cache_position[0] + past_length = past_key_values.get_seq_length() if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -545,7 +532,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py index a62f70d5ea0f..7bf4cf034333 100644 --- a/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py +++ b/src/transformers/models/phi4_multimodal/modeling_phi4_multimodal.py @@ -1245,7 +1245,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -1265,9 +1264,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1309,7 +1306,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: @@ -1322,7 +1318,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/phimoe/modular_phimoe.py b/src/transformers/models/phimoe/modular_phimoe.py index 4ab1b447a03a..b24aedf54c73 100644 --- a/src/transformers/models/phimoe/modular_phimoe.py +++ b/src/transformers/models/phimoe/modular_phimoe.py @@ -367,6 +367,7 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, + cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -382,7 +383,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = past_key_values.get_seq_length() + past_length = cache_position[0] if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -391,6 +392,7 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, + cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/qwen2/modeling_qwen2.py b/src/transformers/models/qwen2/modeling_qwen2.py index 446680dae473..8bd0a380066c 100644 --- a/src/transformers/models/qwen2/modeling_qwen2.py +++ b/src/transformers/models/qwen2/modeling_qwen2.py @@ -209,7 +209,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -223,9 +222,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -362,7 +359,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -374,14 +370,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -390,7 +382,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -413,7 +404,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index b88b8fcc0bed..8e3f846a774a 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -479,7 +479,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -491,14 +490,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -507,7 +502,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -527,7 +521,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3/modeling_qwen3.py b/src/transformers/models/qwen3/modeling_qwen3.py index 5db2cc38df97..8074eb290f7d 100644 --- a/src/transformers/models/qwen3/modeling_qwen3.py +++ b/src/transformers/models/qwen3/modeling_qwen3.py @@ -255,7 +255,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -269,9 +268,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -387,7 +384,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -399,14 +395,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -415,7 +407,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -438,7 +429,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_5/modeling_qwen3_5.py b/src/transformers/models/qwen3_5/modeling_qwen3_5.py index 8fb864689a2c..8a05b8bf7ee9 100644 --- a/src/transformers/models/qwen3_5/modeling_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modeling_qwen3_5.py @@ -137,14 +137,13 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length(layer_idx) kv_length = query_length + past_seen_tokens return kv_length, kv_offset @@ -512,7 +511,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: Qwen3_5DynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) @@ -520,12 +518,7 @@ def forward( # Set up dimensions for reshapes later batch_size, seq_len, _ = hidden_states.shape - use_precomputed_states = ( - cache_params is not None - and cache_params.has_previous_state - and seq_len == 1 - and cache_position is not None - ) + use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1 # getting projected states from cache if it exists if cache_params is not None: @@ -743,7 +736,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -762,9 +754,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -844,7 +834,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -856,7 +845,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) elif self.layer_type == "full_attention": @@ -866,7 +854,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1321,7 +1308,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -1333,15 +1319,11 @@ def forward( if use_cache and past_key_values is None: past_key_values = Qwen3_5DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # mrope: the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -1355,11 +1337,10 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids) @@ -1374,7 +1355,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1385,7 +1365,7 @@ def forward( past_key_values=past_key_values, ) - def _update_linear_attn_mask(self, attention_mask, cache_position): + def _update_linear_attn_mask(self, attention_mask, past_key_values): """ NOTE: Left-padding is used for linear attention mask. No need for zeroing states when @@ -1393,7 +1373,9 @@ def _update_linear_attn_mask(self, attention_mask, cache_position): 2. Attending to all inputs """ linear_attn_mask = attention_mask - if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( + attention_mask is not None and torch.all(attention_mask == 1) + ): linear_attn_mask = None return linear_attn_mask @@ -1644,7 +1626,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3_5ModelOutputWithPast: r""" @@ -1697,7 +1678,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1882,7 +1862,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3_5CausalLMOutputWithPast: @@ -1943,7 +1922,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1972,7 +1950,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1989,7 +1966,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py index 8fbccbd23db1..a8011dc16599 100644 --- a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py +++ b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py @@ -244,14 +244,13 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length(layer_idx) kv_length = query_length + past_seen_tokens return kv_length, kv_offset @@ -513,7 +512,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: Qwen3_5MoeDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) @@ -521,12 +519,7 @@ def forward( # Set up dimensions for reshapes later batch_size, seq_len, _ = hidden_states.shape - use_precomputed_states = ( - cache_params is not None - and cache_params.has_previous_state - and seq_len == 1 - and cache_position is not None - ) + use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1 # getting projected states from cache if it exists if cache_params is not None: @@ -746,7 +739,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -765,9 +757,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -928,7 +918,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -940,7 +929,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) elif self.layer_type == "full_attention": @@ -950,7 +938,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1446,7 +1433,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -1458,15 +1444,11 @@ def forward( if use_cache and past_key_values is None: past_key_values = Qwen3_5MoeDynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # mrope: the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -1480,11 +1462,10 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids) @@ -1499,7 +1480,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1510,7 +1490,7 @@ def forward( past_key_values=past_key_values, ) - def _update_linear_attn_mask(self, attention_mask, cache_position): + def _update_linear_attn_mask(self, attention_mask, past_key_values): """ NOTE: Left-padding is used for linear attention mask. No need for zeroing states when @@ -1518,7 +1498,9 @@ def _update_linear_attn_mask(self, attention_mask, cache_position): 2. Attending to all inputs """ linear_attn_mask = attention_mask - if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( + attention_mask is not None and torch.all(attention_mask == 1) + ): linear_attn_mask = None return linear_attn_mask @@ -1769,7 +1751,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3_5MoeModelOutputWithPast: r""" @@ -1822,7 +1803,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1946,7 +1926,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -2086,7 +2065,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3_5MoeCausalLMOutputWithPast: @@ -2151,7 +2129,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -2195,7 +2172,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -2212,7 +2188,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index 025cb3a8de52..b4e56fdac632 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -159,7 +159,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -173,9 +172,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -645,7 +642,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -685,7 +681,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_next/modeling_qwen3_next.py b/src/transformers/models/qwen3_next/modeling_qwen3_next.py index 2190ec03f83b..060adf4ffbb4 100644 --- a/src/transformers/models/qwen3_next/modeling_qwen3_next.py +++ b/src/transformers/models/qwen3_next/modeling_qwen3_next.py @@ -154,14 +154,13 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length(layer_idx) kv_length = query_length + past_seen_tokens return kv_length, kv_offset @@ -378,7 +377,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -397,9 +395,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -686,7 +682,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: Qwen3NextDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, ): hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) @@ -694,12 +689,7 @@ def forward( # Set up dimensions for reshapes later batch_size, seq_len, _ = hidden_states.shape - use_precomputed_states = ( - cache_params is not None - and cache_params.has_previous_state - and seq_len == 1 - and cache_position is not None - ) + use_precomputed_states = cache_params is not None and cache_params.has_previous_state and seq_len == 1 # getting projected states from cache if it exists if cache_params is not None: @@ -928,7 +918,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -940,7 +929,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) elif self.layer_type == "full_attention": @@ -950,7 +938,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1025,7 +1012,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -1037,23 +1023,19 @@ def forward( if use_cache and past_key_values is None: past_key_values = Qwen3NextDynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds position_embeddings = self.rotary_emb(hidden_states, position_ids) @@ -1068,7 +1050,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1079,7 +1060,7 @@ def forward( past_key_values=past_key_values, ) - def _update_linear_attn_mask(self, attention_mask, cache_position): + def _update_linear_attn_mask(self, attention_mask, past_key_values): """ NOTE: Left-padding is used for linear attention mask. No need for zeroing states when @@ -1087,7 +1068,9 @@ def _update_linear_attn_mask(self, attention_mask, cache_position): 2. Attending to all inputs """ linear_attn_mask = attention_mask - if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( + attention_mask is not None and torch.all(attention_mask == 1) + ): linear_attn_mask = None return linear_attn_mask @@ -1204,7 +1187,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index 95eeca6908fb..068a4f9bc327 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1505,7 +1505,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -1519,9 +1518,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1701,7 +1698,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -1725,15 +1721,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -1747,7 +1739,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -1764,7 +1755,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -2068,7 +2058,6 @@ def forward( use_cache=None, output_router_logits: bool | None = None, use_audio_in_video=None, - cache_position=None, video_second_per_grid=None, **kwargs, ) -> tuple | Qwen3OmniMoeThinkerCausalLMOutputWithPast: @@ -2231,7 +2220,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, deepstack_visual_embeds=visual_embeds_multiscale, visual_pos_masks=visual_pos_masks, **kwargs, @@ -2398,7 +2386,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -2412,9 +2399,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -2602,7 +2587,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if input_ids is not None: @@ -2611,14 +2595,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -2627,7 +2607,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -2648,7 +2627,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -2697,7 +2675,6 @@ def forward( inputs_embeds=None, labels=None, use_cache=None, - cache_position=None, generation_steps=None, **kwargs, ) -> CausalLMOutputWithPast: @@ -2721,7 +2698,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -2957,7 +2933,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -2981,15 +2956,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -3003,7 +2974,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -3020,7 +2990,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -3104,7 +3073,6 @@ def forward( labels=None, use_cache=None, output_router_logits=None, - cache_position=None, residual_codes=None, trailing_text_hidden=None, tts_pad_embed=None, @@ -3169,7 +3137,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) @@ -3255,7 +3222,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, is_first_iteration=False, **kwargs, ): @@ -3265,7 +3231,6 @@ def prepare_inputs_for_generation( past_key_values, attention_mask, inputs_embeds, - cache_position, is_first_iteration=is_first_iteration, **kwargs, ) @@ -3437,7 +3402,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -3451,9 +3415,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -3547,7 +3509,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: """ @@ -3563,8 +3524,6 @@ def forward( If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states - cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): - Indices depicting the position of the input sequence tokens in the sequence kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model @@ -3580,7 +3539,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + self.self_attn_layer_scale(hidden_states) @@ -3626,7 +3584,6 @@ def forward( past_key_values=None, inputs_embeds=None, use_cache=None, - cache_position=None, **kwargs, ) -> BaseModelOutputWithPast: if input_ids is not None: @@ -3640,14 +3597,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -3656,7 +3609,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -3679,7 +3631,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 0208a24ec05d..084d02c96004 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -462,7 +462,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -476,9 +475,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -535,7 +532,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -855,7 +851,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -879,15 +874,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -901,7 +892,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -918,7 +908,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1196,7 +1185,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLModelOutputWithPast: r""" @@ -1278,7 +1266,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, **kwargs, @@ -1384,7 +1371,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLCausalLMOutputWithPast: @@ -1445,7 +1431,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1474,7 +1459,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1491,7 +1475,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index d163e85ae96d..c918f3430b94 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -253,7 +253,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -267,9 +266,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -926,7 +923,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, # args for deepstack visual_pos_masks: torch.Tensor | None = None, deepstack_visual_embeds: list[torch.Tensor] | None = None, @@ -950,15 +946,11 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - # the hard coded `3` is for temporal, height and width. if position_ids is None: - position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) @@ -972,7 +964,6 @@ def forward( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=text_position_ids, ) @@ -989,7 +980,6 @@ def forward( attention_mask=attention_mask, position_ids=text_position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1323,7 +1313,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLMoeModelOutputWithPast: r""" @@ -1405,7 +1394,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, visual_pos_masks=visual_pos_masks, deepstack_visual_embeds=deepstack_visual_embeds, **kwargs, @@ -1564,7 +1552,6 @@ def forward( pixel_values_videos: torch.FloatTensor | None = None, image_grid_thw: torch.LongTensor | None = None, video_grid_thw: torch.LongTensor | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> tuple | Qwen3VLMoeCausalLMOutputWithPast: @@ -1630,7 +1617,6 @@ def forward( attention_mask=attention_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, - cache_position=cache_position, **kwargs, ) @@ -1674,7 +1660,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, pixel_values=None, @@ -1691,7 +1676,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, pixel_values=pixel_values, pixel_values_videos=pixel_values_videos, diff --git a/src/transformers/models/seed_oss/modeling_seed_oss.py b/src/transformers/models/seed_oss/modeling_seed_oss.py index c73268097848..da2498bff21c 100644 --- a/src/transformers/models/seed_oss/modeling_seed_oss.py +++ b/src/transformers/models/seed_oss/modeling_seed_oss.py @@ -188,7 +188,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -202,9 +201,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward diff --git a/src/transformers/models/smollm3/modeling_smollm3.py b/src/transformers/models/smollm3/modeling_smollm3.py index acb7a02e87ee..1b3c1c07ee10 100644 --- a/src/transformers/models/smollm3/modeling_smollm3.py +++ b/src/transformers/models/smollm3/modeling_smollm3.py @@ -221,7 +221,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -236,8 +235,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -390,7 +388,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -402,14 +399,10 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # It may already have been prepared by e.g. `generate` if not isinstance(causal_mask_mapping := attention_mask, dict): @@ -418,7 +411,6 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, "past_key_values": past_key_values, "position_ids": position_ids, } @@ -441,7 +433,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/starcoder2/modeling_starcoder2.py b/src/transformers/models/starcoder2/modeling_starcoder2.py index 503453e7db1c..8b89a1d1745c 100644 --- a/src/transformers/models/starcoder2/modeling_starcoder2.py +++ b/src/transformers/models/starcoder2/modeling_starcoder2.py @@ -162,7 +162,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -176,9 +175,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -359,7 +356,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -371,21 +367,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -403,7 +394,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py index 6212b61bd2a7..bf0bf5af3f74 100644 --- a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py @@ -346,7 +346,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -360,9 +359,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -641,7 +638,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -655,9 +651,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -791,7 +785,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -803,21 +796,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -832,7 +820,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -1008,10 +995,8 @@ def get_audio_features( `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~VoxtralRealtimeFeatureExtractor.__call__`] - padding_cache (`VoxtralRealtimeConv1dPaddingCache`, *optional*): Cache for padding in convolutional layers to maintain state across streaming chunks. - encoder_inputs_embeds (`torch.FloatTensor`, *optional*): Optionally, instead of passing `input_features` you can choose to directly pass an embedded representation for the encoder. """ diff --git a/src/transformers/models/youtu/modeling_youtu.py b/src/transformers/models/youtu/modeling_youtu.py index 59026df304da..f0b4981fe01f 100644 --- a/src/transformers/models/youtu/modeling_youtu.py +++ b/src/transformers/models/youtu/modeling_youtu.py @@ -323,7 +323,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_length = hidden_states.shape[:-1] @@ -356,9 +355,7 @@ def forward( key_states = torch.cat((k_pass, k_rot), dim=-1) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) if is_flash_attention_requested(self.config) and self.qk_head_dim != self.v_head_dim: value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) From c2fde0ab219e65e3b26fed6cae627bf6f6f28ae5 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 20 Feb 2026 19:16:34 +0100 Subject: [PATCH 06/42] a few more --- .../models/bamba/modeling_bamba.py | 3 +- src/transformers/models/doge/modeling_doge.py | 21 +++---------- src/transformers/models/doge/modular_doge.py | 9 +----- .../ernie4_5_moe/modeling_ernie4_5_moe.py | 2 -- .../models/falcon_h1/modeling_falcon_h1.py | 3 +- .../models/flex_olmo/modeling_flex_olmo.py | 16 ++-------- .../models/flex_olmo/modular_flex_olmo.py | 14 ++------- .../models/gpt_oss/modeling_gpt_oss.py | 2 -- .../modeling_granitemoehybrid.py | 3 +- .../models/jamba/modeling_jamba.py | 31 +++++-------------- .../models/jamba/modular_jamba.py | 31 +++++-------------- .../models/minimax/modeling_minimax.py | 17 ++-------- .../models/minimax/modular_minimax.py | 15 ++------- .../models/minimax_m2/modeling_minimax_m2.py | 16 ++-------- .../models/minimax_m2/modular_minimax_m2.py | 12 ++----- .../models/mixtral/modeling_mixtral.py | 16 ++-------- .../models/mixtral/modular_mixtral.py | 16 ++-------- .../models/olmoe/modeling_olmoe.py | 2 -- .../models/phimoe/modeling_phimoe.py | 20 +++--------- .../models/phimoe/modular_phimoe.py | 4 +-- .../models/qwen2_moe/modeling_qwen2_moe.py | 2 -- .../qwen3_5_moe/modeling_qwen3_5_moe.py | 1 - .../models/qwen3_moe/modeling_qwen3_moe.py | 12 ++----- .../models/qwen3_next/modeling_qwen3_next.py | 1 - .../modeling_voxtral_realtime.py | 21 ++----------- .../modular_voxtral_realtime.py | 21 ++----------- 26 files changed, 62 insertions(+), 249 deletions(-) diff --git a/src/transformers/models/bamba/modeling_bamba.py b/src/transformers/models/bamba/modeling_bamba.py index ace73fd37807..a08f1b4be901 100644 --- a/src/transformers/models/bamba/modeling_bamba.py +++ b/src/transformers/models/bamba/modeling_bamba.py @@ -166,10 +166,9 @@ def reorder_cache(self, beam_idx: torch.LongTensor): device = self.ssm_states[layer_idx].device self.ssm_states[layer_idx] = self.ssm_states[layer_idx].index_select(0, beam_idx.to(device)) - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset diff --git a/src/transformers/models/doge/modeling_doge.py b/src/transformers/models/doge/modeling_doge.py index cc32bf7a0e49..4aad59b52a9a 100644 --- a/src/transformers/models/doge/modeling_doge.py +++ b/src/transformers/models/doge/modeling_doge.py @@ -293,7 +293,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -307,9 +306,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # calculate dynamic mask from value_states dt_states = self.dt_proj( @@ -475,7 +472,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: # sequence transformation @@ -488,7 +484,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = F.dropout(hidden_states, p=self.hidden_dropout, training=self.training) @@ -565,7 +560,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -577,20 +571,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -605,7 +595,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -753,7 +742,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, output_router_logits: bool | None = None, **kwargs: Unpack[TransformersKwargs], @@ -792,7 +780,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/doge/modular_doge.py b/src/transformers/models/doge/modular_doge.py index 02a86ad31579..5fe6953e4e06 100644 --- a/src/transformers/models/doge/modular_doge.py +++ b/src/transformers/models/doge/modular_doge.py @@ -329,7 +329,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs, ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: input_shape = hidden_states.shape[:-1] @@ -343,9 +342,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) # calculate dynamic mask from value_states dt_states = self.dt_proj( @@ -499,7 +496,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: # sequence transformation @@ -512,7 +508,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = F.dropout(hidden_states, p=self.hidden_dropout, training=self.training) @@ -676,7 +671,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, output_router_logits: bool | None = None, **kwargs: Unpack[TransformersKwargs], @@ -715,7 +709,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py index 24f05e3f5217..9629c33bee01 100644 --- a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py +++ b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py @@ -688,7 +688,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -712,7 +711,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/falcon_h1/modeling_falcon_h1.py b/src/transformers/models/falcon_h1/modeling_falcon_h1.py index 3bdd8427da30..18396cd1ba16 100644 --- a/src/transformers/models/falcon_h1/modeling_falcon_h1.py +++ b/src/transformers/models/falcon_h1/modeling_falcon_h1.py @@ -174,10 +174,9 @@ def reorder_cache(self, beam_idx: torch.LongTensor): device = self.ssm_states[layer_idx].device self.ssm_states[layer_idx] = self.ssm_states[layer_idx].index_select(0, beam_idx.to(device)) - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset diff --git a/src/transformers/models/flex_olmo/modeling_flex_olmo.py b/src/transformers/models/flex_olmo/modeling_flex_olmo.py index aa771fa55ce4..f43ad61eb87b 100644 --- a/src/transformers/models/flex_olmo/modeling_flex_olmo.py +++ b/src/transformers/models/flex_olmo/modeling_flex_olmo.py @@ -380,7 +380,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> torch.FloatTensor: @@ -392,7 +391,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -466,7 +464,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -478,19 +475,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -508,7 +501,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -632,7 +624,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -673,7 +664,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/flex_olmo/modular_flex_olmo.py b/src/transformers/models/flex_olmo/modular_flex_olmo.py index 54e59a2785fa..6cdde9ad25cf 100644 --- a/src/transformers/models/flex_olmo/modular_flex_olmo.py +++ b/src/transformers/models/flex_olmo/modular_flex_olmo.py @@ -239,7 +239,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> torch.FloatTensor: @@ -251,7 +250,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -292,7 +290,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -304,19 +301,15 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -334,7 +327,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/gpt_oss/modeling_gpt_oss.py b/src/transformers/models/gpt_oss/modeling_gpt_oss.py index 1b0b8b8ca394..891860848763 100644 --- a/src/transformers/models/gpt_oss/modeling_gpt_oss.py +++ b/src/transformers/models/gpt_oss/modeling_gpt_oss.py @@ -626,7 +626,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -666,7 +665,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py index 3b25c2d977b9..d52d6bbb8b07 100644 --- a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py @@ -280,10 +280,9 @@ def reorder_cache(self, beam_idx: torch.LongTensor): device = self.ssm_states[layer_idx].device self.ssm_states[layer_idx] = self.ssm_states[layer_idx].index_select(0, beam_idx.to(device)) - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset diff --git a/src/transformers/models/jamba/modeling_jamba.py b/src/transformers/models/jamba/modeling_jamba.py index c3a72cd916fd..0b45936338ab 100755 --- a/src/transformers/models/jamba/modeling_jamba.py +++ b/src/transformers/models/jamba/modeling_jamba.py @@ -153,10 +153,9 @@ def reorder_cache(self, beam_idx: torch.LongTensor): device = self.ssm_states[layer_idx].device self.ssm_states[layer_idx] = self.ssm_states[layer_idx].index_select(0, beam_idx.to(device)) - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset @@ -262,7 +261,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: HybridMambaAttentionDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -273,9 +271,7 @@ def forward( value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) if past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -696,7 +692,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: HybridMambaAttentionDynamicCache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -707,7 +702,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -813,7 +807,6 @@ def forward( past_key_values: HybridMambaAttentionDynamicCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -830,24 +823,19 @@ def forward( device=inputs_embeds.device, ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - mamba_mask = self._update_mamba_mask(attention_mask, cache_position) + mamba_mask = self._update_mamba_mask(attention_mask, past_key_values) hidden_states = inputs_embeds for decoder_layer in self.layers: layer_mask = mamba_mask if isinstance(decoder_layer, JambaMambaDecoderLayer) else causal_mask @@ -858,7 +846,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -872,14 +859,14 @@ def forward( past_key_values=past_key_values, ) - def _update_mamba_mask(self, attention_mask, cache_position): + def _update_mamba_mask(self, attention_mask, past_key_values): """ No need for zeroing states when 1. Cached forward 2. Attending to all inputs """ mamba_mask = attention_mask - if (cache_position is not None and cache_position[0] > 0) or ( + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( attention_mask is not None and torch.all(attention_mask == 1) ): mamba_mask = None @@ -998,7 +985,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -1038,7 +1024,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/jamba/modular_jamba.py b/src/transformers/models/jamba/modular_jamba.py index 3959dac3a08b..1ff4b31a3a07 100644 --- a/src/transformers/models/jamba/modular_jamba.py +++ b/src/transformers/models/jamba/modular_jamba.py @@ -126,10 +126,9 @@ def reorder_cache(self, beam_idx: torch.LongTensor): device = self.ssm_states[layer_idx].device self.ssm_states[layer_idx] = self.ssm_states[layer_idx].index_select(0, beam_idx.to(device)) - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset @@ -155,7 +154,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: HybridMambaAttentionDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -166,9 +164,7 @@ def forward( value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) if past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -541,7 +537,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: HybridMambaAttentionDynamicCache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.FloatTensor: residual = hidden_states @@ -552,7 +547,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -658,7 +652,6 @@ def forward( past_key_values: HybridMambaAttentionDynamicCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -675,24 +668,19 @@ def forward( device=inputs_embeds.device, ) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - mamba_mask = self._update_mamba_mask(attention_mask, cache_position) + mamba_mask = self._update_mamba_mask(attention_mask, past_key_values) hidden_states = inputs_embeds for decoder_layer in self.layers: layer_mask = mamba_mask if isinstance(decoder_layer, JambaMambaDecoderLayer) else causal_mask @@ -703,7 +691,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -717,14 +704,14 @@ def forward( past_key_values=past_key_values, ) - def _update_mamba_mask(self, attention_mask, cache_position): + def _update_mamba_mask(self, attention_mask, past_key_values): """ No need for zeroing states when 1. Cached forward 2. Attending to all inputs """ mamba_mask = attention_mask - if (cache_position is not None and cache_position[0] > 0) or ( + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( attention_mask is not None and torch.all(attention_mask == 1) ): mamba_mask = None @@ -746,7 +733,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -780,7 +766,6 @@ def forward( inputs_embeds, labels, use_cache, - cache_position, logits_to_keep, **kwargs, ) diff --git a/src/transformers/models/minimax/modeling_minimax.py b/src/transformers/models/minimax/modeling_minimax.py index eefdced42b66..d6eaab3426c5 100644 --- a/src/transformers/models/minimax/modeling_minimax.py +++ b/src/transformers/models/minimax/modeling_minimax.py @@ -166,7 +166,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_len, hidden_size = hidden_states.shape @@ -562,7 +561,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: hidden_states = self.input_layernorm(hidden_states) @@ -574,7 +572,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual * self.attn_alpha_factor + hidden_states * self.attn_beta_factor @@ -650,7 +647,6 @@ def forward( past_key_values: MiniMaxCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -666,20 +662,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -701,7 +693,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -825,7 +816,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -865,7 +855,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/minimax/modular_minimax.py b/src/transformers/models/minimax/modular_minimax.py index 15de27f09dd3..549d4f13433e 100644 --- a/src/transformers/models/minimax/modular_minimax.py +++ b/src/transformers/models/minimax/modular_minimax.py @@ -355,7 +355,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: batch_size, seq_len, hidden_size = hidden_states.shape @@ -491,7 +490,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[FlashAttentionKwargs], ) -> tuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None]: hidden_states = self.input_layernorm(hidden_states) @@ -503,7 +501,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) hidden_states = residual * self.attn_alpha_factor + hidden_states * self.attn_beta_factor @@ -545,7 +542,6 @@ def forward( past_key_values: MiniMaxCache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -561,20 +557,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -596,7 +588,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/minimax_m2/modeling_minimax_m2.py b/src/transformers/models/minimax_m2/modeling_minimax_m2.py index 9abba971208b..d19274262810 100644 --- a/src/transformers/models/minimax_m2/modeling_minimax_m2.py +++ b/src/transformers/models/minimax_m2/modeling_minimax_m2.py @@ -375,7 +375,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -386,7 +385,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -458,7 +456,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -470,20 +467,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # No sliding window opposed to mixtral causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -498,7 +491,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -623,7 +615,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -663,7 +654,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/minimax_m2/modular_minimax_m2.py b/src/transformers/models/minimax_m2/modular_minimax_m2.py index 13a656c7f218..9658ac0838d1 100644 --- a/src/transformers/models/minimax_m2/modular_minimax_m2.py +++ b/src/transformers/models/minimax_m2/modular_minimax_m2.py @@ -283,7 +283,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -295,20 +294,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) # No sliding window opposed to mixtral causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -323,7 +318,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/mixtral/modeling_mixtral.py b/src/transformers/models/mixtral/modeling_mixtral.py index 484077d48115..3c75687c4c49 100644 --- a/src/transformers/models/mixtral/modeling_mixtral.py +++ b/src/transformers/models/mixtral/modeling_mixtral.py @@ -369,7 +369,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -380,7 +379,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -450,7 +448,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -462,20 +459,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -490,7 +483,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -615,7 +607,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -655,7 +646,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/mixtral/modular_mixtral.py b/src/transformers/models/mixtral/modular_mixtral.py index 34996ad8a404..2ec3d29a999b 100644 --- a/src/transformers/models/mixtral/modular_mixtral.py +++ b/src/transformers/models/mixtral/modular_mixtral.py @@ -239,7 +239,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -250,7 +249,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -288,7 +286,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -300,20 +297,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -328,7 +321,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -361,7 +353,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -401,7 +392,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index 38d3f6c95411..8a83315a5820 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -631,7 +631,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -672,7 +671,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/phimoe/modeling_phimoe.py b/src/transformers/models/phimoe/modeling_phimoe.py index d93e8bf7cee7..23bc944c522a 100644 --- a/src/transformers/models/phimoe/modeling_phimoe.py +++ b/src/transformers/models/phimoe/modeling_phimoe.py @@ -561,7 +561,6 @@ def forward( attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -572,7 +571,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, **kwargs, ) hidden_states = residual + hidden_states @@ -642,7 +640,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -654,20 +651,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -682,7 +675,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -807,7 +799,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -847,7 +838,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) @@ -888,7 +878,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -904,7 +893,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = cache_position[0] + past_length = past_key_values.get_seq_length() if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -913,7 +902,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/phimoe/modular_phimoe.py b/src/transformers/models/phimoe/modular_phimoe.py index b24aedf54c73..4ab1b447a03a 100644 --- a/src/transformers/models/phimoe/modular_phimoe.py +++ b/src/transformers/models/phimoe/modular_phimoe.py @@ -367,7 +367,6 @@ def prepare_inputs_for_generation( past_key_values=None, attention_mask=None, inputs_embeds=None, - cache_position=None, position_ids=None, use_cache=True, logits_to_keep=None, @@ -383,7 +382,7 @@ def prepare_inputs_for_generation( and hasattr(self.config, "original_max_position_embeddings") and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 ): - past_length = cache_position[0] + past_length = past_key_values.get_seq_length() if past_length <= self.config.original_max_position_embeddings: past_key_values = None @@ -392,7 +391,6 @@ def prepare_inputs_for_generation( past_key_values=past_key_values, attention_mask=attention_mask, inputs_embeds=inputs_embeds, - cache_position=cache_position, position_ids=position_ids, use_cache=use_cache, logits_to_keep=logits_to_keep, diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index 8e3f846a774a..e3464bedefa2 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -644,7 +644,6 @@ def forward( labels: torch.LongTensor | None = None, use_cache: bool | None = None, output_router_logits: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> MoeCausalLMOutputWithPast: @@ -684,7 +683,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py index a8011dc16599..0dba118b1719 100644 --- a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py +++ b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py @@ -1965,7 +1965,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index b4e56fdac632..d63882215609 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -477,7 +477,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> MoeModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -489,20 +488,16 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -517,7 +512,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) diff --git a/src/transformers/models/qwen3_next/modeling_qwen3_next.py b/src/transformers/models/qwen3_next/modeling_qwen3_next.py index 060adf4ffbb4..2506e9fb3727 100644 --- a/src/transformers/models/qwen3_next/modeling_qwen3_next.py +++ b/src/transformers/models/qwen3_next/modeling_qwen3_next.py @@ -1226,7 +1226,6 @@ def forward( inputs_embeds=inputs_embeds, use_cache=use_cache, output_router_logits=output_router_logits, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py index bf0bf5af3f74..5565bee83d54 100644 --- a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py @@ -432,7 +432,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -453,7 +452,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -536,7 +534,6 @@ def forward( past_key_values: Cache | None = None, padding_cache: VoxtralRealtimeConv1dPaddingCache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, use_padding_cache: bool | None = None, attention_mask: torch.Tensor | None = None, @@ -560,21 +557,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -589,7 +581,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -707,7 +698,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, t_cond: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -721,7 +711,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -856,7 +845,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -884,7 +872,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1034,7 +1021,6 @@ def forward( encoder_inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, num_delay_tokens: int | torch.Tensor = None, **kwargs: Unpack[TransformersKwargs], @@ -1114,7 +1100,6 @@ def forward( inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, t_cond=t_cond, **kwargs, diff --git a/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py index 8dc007c8daaa..b4efbdd23dbe 100644 --- a/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py @@ -277,7 +277,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -298,7 +297,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -369,7 +367,6 @@ def forward( past_key_values: Cache | None = None, padding_cache: VoxtralRealtimeConv1dPaddingCache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, use_padding_cache: bool | None = None, attention_mask: torch.Tensor | None = None, @@ -393,21 +390,16 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) mask_function = create_causal_mask if self.config.sliding_window is None else create_sliding_window_causal_mask causal_mask = mask_function( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -422,7 +414,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -467,7 +458,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, t_cond: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], @@ -481,7 +471,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -516,7 +505,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -544,7 +532,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -659,7 +646,6 @@ def forward( encoder_inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, num_delay_tokens: int | torch.Tensor = None, **kwargs: Unpack[TransformersKwargs], @@ -739,7 +725,6 @@ def forward( inputs_embeds=inputs_embeds, labels=labels, use_cache=use_cache, - cache_position=cache_position, logits_to_keep=logits_to_keep, t_cond=t_cond, **kwargs, From 0f12a85bb4b2f9505f1712d76a3703495522195f Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Mon, 23 Feb 2026 10:37:30 +0100 Subject: [PATCH 07/42] style --- .../higgs_audio_v2/modeling_higgs_audio_v2.py | 24 ++++--------------- .../higgs_audio_v2/modular_higgs_audio_v2.py | 19 ++++----------- .../models/zamba/modeling_zamba.py | 3 +-- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py index f760bc611f80..a0f106167721 100644 --- a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py +++ b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py @@ -185,7 +185,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -199,9 +198,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -247,7 +244,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -272,7 +268,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -432,7 +427,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -538,20 +532,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -566,7 +555,6 @@ def forward( audio_token_mask=audio_token_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -636,7 +624,7 @@ def prepare_inputs_for_generation( model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs) if audio_input_ids is not None and model_inputs.get("past_key_values") is not None: - current_cache_length = model_inputs["cache_position"][0] + current_cache_length = model_inputs.get("past_key_values").get_seq_length() audio_token_mask = (input_ids == self.config.audio_token_id) | ( input_ids == self.config.audio_delay_token_id ) @@ -672,7 +660,6 @@ def forward( labels: torch.LongTensor | None = None, audio_labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ): @@ -761,7 +748,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/higgs_audio_v2/modular_higgs_audio_v2.py b/src/transformers/models/higgs_audio_v2/modular_higgs_audio_v2.py index d7bbce5248b2..295d7c4eb77e 100644 --- a/src/transformers/models/higgs_audio_v2/modular_higgs_audio_v2.py +++ b/src/transformers/models/higgs_audio_v2/modular_higgs_audio_v2.py @@ -232,7 +232,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: residual = hidden_states @@ -257,7 +256,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -340,7 +338,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, - cache_position: torch.LongTensor | None = None, use_cache: bool | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: @@ -446,20 +443,15 @@ def forward( if use_cache and past_key_values is None: past_key_values = DynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position: torch.Tensor = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, inputs_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) @@ -474,7 +466,6 @@ def forward( audio_token_mask=audio_token_mask, position_ids=position_ids, past_key_values=past_key_values, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -518,7 +509,7 @@ def prepare_inputs_for_generation( model_inputs = super().prepare_inputs_for_generation(input_ids, **kwargs) if audio_input_ids is not None and model_inputs.get("past_key_values") is not None: - current_cache_length = model_inputs["cache_position"][0] + current_cache_length = model_inputs.get("past_key_values").get_seq_length() audio_token_mask = (input_ids == self.config.audio_token_id) | ( input_ids == self.config.audio_delay_token_id ) @@ -554,7 +545,6 @@ def forward( labels: torch.LongTensor | None = None, audio_labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ): @@ -643,7 +633,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/zamba/modeling_zamba.py b/src/transformers/models/zamba/modeling_zamba.py index 2a59ad3901ec..1d427382ba2a 100644 --- a/src/transformers/models/zamba/modeling_zamba.py +++ b/src/transformers/models/zamba/modeling_zamba.py @@ -173,10 +173,9 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return self.key_cache[layer_idx].shape[-2] # Copied from transformers.models.jamba.modeling_jamba.HybridMambaAttentionDynamicCache.get_mask_sizes - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset From b2d91fcc400c7a4c623d4bd3ee5d16b354315d8b Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 11:15:12 +0100 Subject: [PATCH 08/42] style --- docs/source/en/model_doc/colmodernvbert.md | 3 +-- docs/source/en/model_doc/modernvbert.md | 3 +-- .../models/voxtral_realtime/modular_voxtral_realtime.py | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/source/en/model_doc/colmodernvbert.md b/docs/source/en/model_doc/colmodernvbert.md index 5d06662be95b..92f5aa3f4f97 100755 --- a/docs/source/en/model_doc/colmodernvbert.md +++ b/docs/source/en/model_doc/colmodernvbert.md @@ -13,8 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was released on 2025-10-01 and added to Hugging Face Transformers on .* -*This model was released on 2025-10-01 and added to Hugging Face Transformers on .* +*This model was released on 2025-10-01 and added to Hugging Face Transformers on 2026-02-23.* # ColModernVBert diff --git a/docs/source/en/model_doc/modernvbert.md b/docs/source/en/model_doc/modernvbert.md index c48ce4c793b8..d55c6a178ad8 100755 --- a/docs/source/en/model_doc/modernvbert.md +++ b/docs/source/en/model_doc/modernvbert.md @@ -13,8 +13,7 @@ specific language governing permissions and limitations under the License. rendered properly in your Markdown viewer. --> -*This model was released on 2025-10-01 and added to Hugging Face Transformers on .* -*This model was released on 2025-10-01 and added to Hugging Face Transformers on .* +*This model was released on 2025-10-01 and added to Hugging Face Transformers on 2026-02-23.* # ModernVBert diff --git a/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py index 7b396b50391f..49dbdf5b6d0d 100644 --- a/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modular_voxtral_realtime.py @@ -546,10 +546,8 @@ def get_audio_features( `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~VoxtralRealtimeFeatureExtractor.__call__`] - padding_cache (`VoxtralRealtimeConv1dPaddingCache`, *optional*): Cache for padding in convolutional layers to maintain state across streaming chunks. - encoder_inputs_embeds (`torch.FloatTensor`, *optional*): Optionally, instead of passing `input_features` you can choose to directly pass an embedded representation for the encoder. """ From 49b47fed89b4f0ae088ecfddce29a2aaed47ff1d Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 11:21:30 +0100 Subject: [PATCH 09/42] better get_seq_length --- src/transformers/cache_utils.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 0eb3e05012b7..73208bceeac3 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -337,9 +337,7 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: def get_seq_length(self) -> int: """Returns the sequence length of the cached states.""" - # Occupied cache == any slot in the 3rd dim (sequence length) holds a non-zero value. To save on compute, let's - # limit the check to the first batch member and head dimension. - return (self.keys[0, 0].any(dim=-1)).sum() if self.is_initialized else 0 + return self.cumulative_length def get_max_cache_shape(self) -> int: """Return the maximum cache shape of the cache""" @@ -458,10 +456,6 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: return kv_length, kv_offset - def get_seq_length(self) -> int: - """Returns the sequence length of the cached states.""" - return self.cumulative_length - class QuantizedLayer(DynamicLayer): """ From 94a44d0cce7f755b4055ba966ee8eff062193e57 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 12:06:50 +0100 Subject: [PATCH 10/42] fix signatures --- src/transformers/masking_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index 4370bd6faf4d..7f2f7a5f1925 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -857,7 +857,8 @@ def create_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, # not used anymore but kept for BC + cache_position: torch.Tensor | None = None, # not used anymore but kept for BC + *, past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -1061,7 +1062,8 @@ def create_sliding_window_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, # not used anymore but kept for BC + cache_position: torch.Tensor | None = None, # not used anymore but kept for BC + *, past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, @@ -1257,7 +1259,8 @@ def create_chunked_causal_mask( config: PreTrainedConfig, inputs_embeds: torch.Tensor, attention_mask: torch.Tensor | None, - cache_position: torch.Tensor, # not used anymore but kept for BC + cache_position: torch.Tensor | None = None, # not used anymore but kept for BC + *, past_key_values: Cache | None, position_ids: torch.Tensor | None = None, or_mask_function: Callable | None = None, From 417da7a1a87ed074d232c8772adaf42a7800c5b8 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 14:54:37 +0100 Subject: [PATCH 11/42] fix device --- src/transformers/generation/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 9c2872b66a79..9155df4a614f 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -563,8 +563,8 @@ def prepare_inputs_for_generation( causal_mask_creation_function = getattr(self, "create_masks_for_generate", create_masks_for_generate) attention_mask = causal_mask_creation_function( config=self.config, - # we only need batch size, seq_length and dtype here - we don't care about the values of the embeddings - inputs_embeds=torch.empty((batch_size, sequence_length), dtype=self.dtype), + # we only need batch size, seq_length, dtype and device here - so we pass a 0-sized tensor with only the metadata + inputs_embeds=torch.empty((batch_size, sequence_length, 0), dtype=self.dtype, device=input_ids.device), attention_mask=attention_mask, cache_position=model_inputs.get("cache_position"), past_key_values=model_inputs.get("past_key_values"), From beffc18cf8492310f2cd6c7def692dcf1d85c7dd Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 15:40:09 +0100 Subject: [PATCH 12/42] win against dynamo --- src/transformers/cache_utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 73208bceeac3..cf95ac11756c 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -72,7 +72,11 @@ def reset(self) -> None: self.values.zero_() # This attribute is set on several Layers if hasattr(self, "cumulative_length"): - self.cumulative_length = 0 + # It can either be an int for dynamic layers, or a tensor for static layers + if isinstance(self.cumulative_length, int): + self.cumulative_length = 0 + else: + self.cumulative_length.zero_() def reorder_cache(self, beam_idx: torch.LongTensor) -> None: """Reorders this layer's cache for beam search.""" @@ -256,7 +260,8 @@ class StaticLayer(CacheLayerMixin): def __init__(self, max_cache_len: int): super().__init__() self.max_cache_len = max_cache_len - self.cumulative_length = 0 + # Very important that it's a tensor here, to avoid recompiling when we update it + self.cumulative_length = torch.tensor([0], dtype=int) def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tensor) -> None: """ @@ -287,6 +292,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens dtype=self.dtype, device=self.device, ) + self.cumulative_length = self.cumulative_length.to(self.device) # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, preventing compiled graph # breaks when updating the cache. However, it is not supported when tracing the graph, so we skip it in this case. # As prefill should never be compiled, this is not an issue and it will still be run (except when users compile @@ -337,7 +343,7 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: def get_seq_length(self) -> int: """Returns the sequence length of the cached states.""" - return self.cumulative_length + return self.cumulative_length.item() def get_max_cache_shape(self) -> int: """Return the maximum cache shape of the cache""" @@ -362,6 +368,7 @@ class StaticSlidingWindowLayer(StaticLayer): def __init__(self, max_cache_len: int, sliding_window: int): effective_max_cache_len = min(sliding_window, max_cache_len) super().__init__(max_cache_len=effective_max_cache_len) + # Here it cannot be a tensor as otherwise we have data-dependent control flows self.cumulative_length = 0 def update( @@ -456,6 +463,10 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: return kv_length, kv_offset + def get_seq_length(self) -> int: + """Returns the sequence length of the cached states.""" + return self.cumulative_length + class QuantizedLayer(DynamicLayer): """ From f15531f02c6743e1c921716400dc61f31bfa2847 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 16:18:38 +0100 Subject: [PATCH 13/42] compatible with cuda graphs --- src/transformers/cache_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index cf95ac11756c..0af586b3cabb 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -323,7 +323,9 @@ def update( # Create a tensor to slice the static kv at the correct indices kv_length = key_states.shape[-2] cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length - self.cumulative_length += kv_length + # Note: it is very important to update the cumulative length in-place, as otherwise we cannot use cudagraphs + # with mode="reduce_overhead" (it complains about overwriting the tensor) + self.cumulative_length.add_(kv_length) # Update the cache try: From 3102c0b81ff6fd4f7c976b4128b5559a808f4af0 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 16:26:52 +0100 Subject: [PATCH 14/42] add args to absorb --- src/transformers/cache_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 0af586b3cabb..9975f4acdce1 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -41,7 +41,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens @abstractmethod def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: ... @abstractmethod @@ -100,7 +100,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self.is_initialized = True def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -182,7 +182,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self._sliding_window_tensor = self._sliding_window_tensor.to(self.device) def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -304,7 +304,7 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens self.is_initialized = True def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -374,7 +374,7 @@ def __init__(self, max_cache_len: int, sliding_window: int): self.cumulative_length = 0 def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -499,7 +499,7 @@ def __init__( self.cumulative_length = 0 def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Update the key and value caches in-place, and return the necessary keys and value states. @@ -735,7 +735,7 @@ def offload(self, layer_idx: int, only_non_sliding: bool = True): self.layers[layer_idx].offload() def update( - self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, **kwargs + self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, *args, **kwargs ) -> tuple[torch.Tensor, torch.Tensor]: """ Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. @@ -761,7 +761,7 @@ def update( torch.cuda.default_stream(key_states.device).wait_stream(self.prefetch_stream) self.prefetch(layer_idx + 1, self.only_non_sliding) - keys, values = self.layers[layer_idx].update(key_states, value_states, **kwargs) + keys, values = self.layers[layer_idx].update(key_states, value_states, *args, **kwargs) if self.offloading: self.offload(layer_idx, self.only_non_sliding) From 5ce0c8ad7d04a59515ef9e53d413bdf584256881 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 17:06:39 +0100 Subject: [PATCH 15/42] fix lfm2 --- src/transformers/cache_utils.py | 6 ++++-- src/transformers/models/lfm2/modeling_lfm2.py | 2 +- src/transformers/models/lfm2/modular_lfm2.py | 2 +- src/transformers/models/lfm2_moe/modeling_lfm2_moe.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 9975f4acdce1..9e379591936c 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -292,7 +292,9 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens dtype=self.dtype, device=self.device, ) - self.cumulative_length = self.cumulative_length.to(self.device) + # We use this check to easily inherit the method in StaticSlidingWindowLayer without overriding to remove this line + if isinstance(self.cumulative_length, torch.Tensor): + self.cumulative_length = self.cumulative_length.to(self.device) # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, preventing compiled graph # breaks when updating the cache. However, it is not supported when tracing the graph, so we skip it in this case. # As prefill should never be compiled, this is not an issue and it will still be run (except when users compile @@ -370,7 +372,7 @@ class StaticSlidingWindowLayer(StaticLayer): def __init__(self, max_cache_len: int, sliding_window: int): effective_max_cache_len = min(sliding_window, max_cache_len) super().__init__(max_cache_len=effective_max_cache_len) - # Here it cannot be a tensor as otherwise we have data-dependent control flows + # Here it cannot be a tensor as otherwise we have data-dependent control flows - overwrite it self.cumulative_length = 0 def update( diff --git a/src/transformers/models/lfm2/modeling_lfm2.py b/src/transformers/models/lfm2/modeling_lfm2.py index c42eb81064ef..c2da10284923 100644 --- a/src/transformers/models/lfm2/modeling_lfm2.py +++ b/src/transformers/models/lfm2/modeling_lfm2.py @@ -502,7 +502,7 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() + cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2/modular_lfm2.py b/src/transformers/models/lfm2/modular_lfm2.py index ff0f66554024..279cc8a55b4c 100644 --- a/src/transformers/models/lfm2/modular_lfm2.py +++ b/src/transformers/models/lfm2/modular_lfm2.py @@ -337,7 +337,7 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() + cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py index b0b50ddbcf40..de29e5055f16 100644 --- a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py @@ -578,7 +578,7 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(x.shape[1], device=x.device) + past_key_values.get_seq_length() + cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) From 318f621522c9f1a54e7bae04ee3a7bc3317743e6 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 17:31:50 +0100 Subject: [PATCH 16/42] fix executorch --- src/transformers/integrations/executorch.py | 9 +++++++++ tests/utils/test_cache_utils.py | 10 +++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/transformers/integrations/executorch.py b/src/transformers/integrations/executorch.py index 27b312377c91..15f8bed97c72 100644 --- a/src/transformers/integrations/executorch.py +++ b/src/transformers/integrations/executorch.py @@ -557,6 +557,9 @@ def forward( The adapter matches the model's forward signature with that in `executorch/extension/llm/runner`, ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box. """ + # Start by resetting static cache + self.static_cache.reset() + past_key_values = self.static_cache outs = self.model( @@ -715,6 +718,9 @@ def forward( Returns: torch.Tensor: Logits output from the model. """ + # Start by resetting static cache + self.cache.reset() + # Forward pass with the model outputs = self.model( input_ids=input_ids, @@ -843,6 +849,9 @@ def __init__(self, model, max_static_cache_length, batch_size): self.register_buffer(f"value_cache_{i}", self.static_cache.layers[i].values, persistent=False) def forward(self, decoder_input_ids, encoder_hidden_states, cache_position): + # Start by resetting static cache + self.cache.reset() + # Get outputs from decoder outputs = self.decoder( input_ids=decoder_input_ids, diff --git a/tests/utils/test_cache_utils.py b/tests/utils/test_cache_utils.py index 95647fc51d15..17fab67baddb 100644 --- a/tests/utils/test_cache_utils.py +++ b/tests/utils/test_cache_utils.py @@ -843,14 +843,15 @@ def setUp(self): def test_static_cache_out_of_bounds(self): """Test StaticCache raises IndexError for out-of-bounds positions.""" static_cache = StaticCache(config=self.config, max_cache_len=self.max_cache_len) - pos_out_of_bounds = torch.tensor([self.max_cache_len]) # Position >= max_cache_len + prefill = torch.tensor([1.0, 2.0, 3.0, 4.0])[None, None, :, None] + # Fill-up the cache + static_cache.update(key_states=prefill, value_states=prefill, layer_idx=0) with self.assertRaises(IndexError): static_cache.update( key_states=torch.tensor([[[[1.0]]]]), value_states=torch.tensor([[[[1.0]]]]), layer_idx=0, - cache_kwargs={"cache_position": pos_out_of_bounds}, ) def test_static_cache(self): @@ -865,13 +866,12 @@ def test_static_cache(self): """ # Scenario 1: Fill up to near capacity static_cache = StaticCache(config=self.config, max_cache_len=self.max_cache_len) - prefill = torch.tensor([1.0, 2.0, 0.0, 0.0])[None, None, :, None] - static_cache.update(key_states=prefill, value_states=prefill, layer_idx=0, cache_kwargs=None) + prefill = torch.tensor([1.0, 2.0])[None, None, :, None] + static_cache.update(key_states=prefill, value_states=prefill, layer_idx=0) static_cache.update( key_states=torch.tensor(3.0)[None, None, None, None], value_states=torch.tensor(3.0)[None, None, None, None], layer_idx=0, - cache_kwargs={"cache_position": torch.tensor([2])}, ) self.assertEqual( static_cache.layers[0].keys[0, 0, :, 0].tolist(), [1.0, 2.0, 3.0, 0.0], "StaticCache Scenario 1 failed" From 0cf77d64d0369507b71f5b58c0aa78b47de562ec Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 17:44:50 +0100 Subject: [PATCH 17/42] fix zamba2 --- src/transformers/models/zamba2/modeling_zamba2.py | 15 +-------------- src/transformers/models/zamba2/modular_zamba2.py | 15 +-------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/src/transformers/models/zamba2/modeling_zamba2.py b/src/transformers/models/zamba2/modeling_zamba2.py index 05f9b9090933..13e3a0e617e9 100644 --- a/src/transformers/models/zamba2/modeling_zamba2.py +++ b/src/transformers/models/zamba2/modeling_zamba2.py @@ -180,25 +180,12 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset - def update_conv_state( - self, layer_idx: int, new_conv_state: torch.Tensor, cache_position: torch.LongTensor - ) -> torch.Tensor: - conv_state = self.conv_states[layer_idx] - cache_position = cache_position.clamp(0, self.conv_kernel_size - 1) - - conv_state = conv_state.roll(shifts=-1, dims=-1) - conv_state[:, :, cache_position] = new_conv_state.to(conv_state.device) - self.conv_states[layer_idx].zero_() - self.conv_states[layer_idx] += conv_state - return self.conv_states[layer_idx] - def reset(self): self.conv_states.zero_() self.ssm_states.zero_() diff --git a/src/transformers/models/zamba2/modular_zamba2.py b/src/transformers/models/zamba2/modular_zamba2.py index 6e557fdf3cc2..58b303726055 100644 --- a/src/transformers/models/zamba2/modular_zamba2.py +++ b/src/transformers/models/zamba2/modular_zamba2.py @@ -122,18 +122,6 @@ def __init__( self.key_cache = [torch.tensor([[]] * batch_size, device=device) for _ in range(config.num_hidden_layers)] self.value_cache = [torch.tensor([[]] * batch_size, device=device) for _ in range(config.num_hidden_layers)] - def update_conv_state( - self, layer_idx: int, new_conv_state: torch.Tensor, cache_position: torch.LongTensor - ) -> torch.Tensor: - conv_state = self.conv_states[layer_idx] - cache_position = cache_position.clamp(0, self.conv_kernel_size - 1) - - conv_state = conv_state.roll(shifts=-1, dims=-1) - conv_state[:, :, cache_position] = new_conv_state.to(conv_state.device) - self.conv_states[layer_idx].zero_() - self.conv_states[layer_idx] += conv_state - return self.conv_states[layer_idx] - def reset(self): self.conv_states.zero_() self.ssm_states.zero_() @@ -146,10 +134,9 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset From e86122ea0cac441f225aea85d7a8b2f566b60f6f Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 18:44:24 +0100 Subject: [PATCH 18/42] fix encoder-decoder --- src/transformers/masking_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index 7f2f7a5f1925..cd1b26c82d06 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -764,6 +764,7 @@ def _preprocess_mask_arguments( past_key_values: Cache | None, position_ids: torch.Tensor | None, layer_idx: int | None, + encoder_hidden_states: torch.Tensor | None = None, ) -> tuple[bool, torch.Tensor | BlockMask | None, int, int]: """ Perform some common pre-processing of the mask arguments we get from the modeling code. Mostly determine the @@ -829,7 +830,9 @@ def _preprocess_mask_arguments( q_offset = 0 # 1. Rely on input directly if attention_mask is None: - kv_length, kv_offset = q_length, 0 + # For encoder-decoders, use encoder_hidden_states to infer kv_length if provided + kv_length = encoder_hidden_states.shape[1] if encoder_hidden_states is not None else q_length + kv_offset = 0 # 2. Rely on the mask instead - needed for special cases like prefix tuning in PEFT # # This is a very unique and special case where an encoder utilizes a cache and expects its length @@ -1002,14 +1005,14 @@ def create_bidirectional_mask( An optional mask function to combine with the base mask function (by doing the intersection of both). This is useful to easily overlay another mask on top, for example for image tokens handling. """ - embeds = encoder_hidden_states if encoder_hidden_states is not None else inputs_embeds # We ignore a few irrelevant arguments at the end as we do not have a (growing) cache here early_exit, attention_mask, _, q_length, kv_length, q_offset, kv_offset = _preprocess_mask_arguments( - config, embeds, attention_mask, None, None, 0 + config, inputs_embeds, attention_mask, None, None, 0, encoder_hidden_states ) if early_exit: return attention_mask + embeds = encoder_hidden_states if encoder_hidden_states is not None else inputs_embeds batch_size, dtype, device = embeds.shape[0], embeds.dtype, embeds.device mask_factory_function = bidirectional_mask_function mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation] From 4b21a5766e5d6e0e8701c2d379e912562590960e Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Tue, 24 Feb 2026 19:00:21 +0100 Subject: [PATCH 19/42] fix --- .../granitemoehybrid/modeling_granitemoehybrid.py | 10 +++++----- .../granitemoehybrid/modular_granitemoehybrid.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py index d52d6bbb8b07..ef724ae7dbca 100644 --- a/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modeling_granitemoehybrid.py @@ -1316,11 +1316,11 @@ def forward( position_ids = cache_position.unsqueeze(0) causal_mask = create_causal_mask( - self.config, - inputs_embeds, - attention_mask, - cache_position, - past_key_values, + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, ) mamba_mask = self._update_mamba_mask(attention_mask, cache_position) diff --git a/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py b/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py index 9b9bd65bf9b0..242232d69227 100644 --- a/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py +++ b/src/transformers/models/granitemoehybrid/modular_granitemoehybrid.py @@ -242,11 +242,11 @@ def forward( position_ids = cache_position.unsqueeze(0) causal_mask = create_causal_mask( - self.config, - inputs_embeds, - attention_mask, - cache_position, - past_key_values, + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, ) mamba_mask = self._update_mamba_mask(attention_mask, cache_position) From 367f44db8abd8349df243a9c580872fa403dde70 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 11:17:54 +0100 Subject: [PATCH 20/42] fix t5gemma --- .../models/t5gemma/modeling_t5gemma.py | 56 +++--------------- .../models/t5gemma/modular_t5gemma.py | 58 ++++--------------- .../models/t5gemma2/modeling_t5gemma2.py | 13 ----- 3 files changed, 18 insertions(+), 109 deletions(-) diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index 2091cf421dfd..942bc480f988 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -618,30 +618,6 @@ def _shift_right(self, input_ids): return shifted_input_ids -def bidirectional_mask_function(attention_mask: torch.Tensor | None) -> Callable: - """ - This creates bidirectional attention mask. - """ - - def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - if attention_mask is None: - return torch.ones((), dtype=torch.bool) - return attention_mask[batch_idx, kv_idx].to(torch.bool) - - return inner_mask - - -def sliding_window_bidirectional_mask_function(sliding_window: int) -> Callable: - """ - This creates bidirectional attention mask with sliding window. - """ - - def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - return (q_idx - sliding_window < kv_idx) & (kv_idx < q_idx + sliding_window) - - return inner_mask - - def make_default_2d_attention_mask( token_ids: torch.LongTensor | None, hidden_states: torch.Tensor, @@ -715,20 +691,10 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": position_ids, } self_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(attention_mask), - ), - "sliding_attention": create_sliding_window_causal_mask( - **mask_kwargs, - or_mask_function=sliding_window_bidirectional_mask_function(self.config.sliding_window), - and_mask_function=bidirectional_mask_function(attention_mask), - ), + "full_attention": create_bidirectional_mask(**mask_kwargs), + "sliding_attention": create_bidirectional_sliding_window_mask(**mask_kwargs), } hidden_states = inputs_embeds @@ -832,19 +798,13 @@ def forward( } if not isinstance(cross_attn_mask_mapping := encoder_attention_mask, dict): - mask_kwargs = { - "config": self.config, - "inputs_embeds": encoder_hidden_states, - "attention_mask": encoder_attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": None, - } cross_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(encoder_attention_mask), - ), + "full_attention": create_bidirectional_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=encoder_attention_mask, + encoder_hidden_states=encoder_hidden_states, + ) } hidden_states = inputs_embeds diff --git a/src/transformers/models/t5gemma/modular_t5gemma.py b/src/transformers/models/t5gemma/modular_t5gemma.py index 7cff24b84793..10cdf4e9e3dd 100644 --- a/src/transformers/models/t5gemma/modular_t5gemma.py +++ b/src/transformers/models/t5gemma/modular_t5gemma.py @@ -50,6 +50,8 @@ Gemma2PreTrainedModel, Gemma2RMSNorm, Gemma2RotaryEmbedding, + create_bidirectional_mask, + create_bidirectional_sliding_window_mask, create_causal_mask, create_sliding_window_causal_mask, eager_attention_forward, @@ -398,30 +400,6 @@ def forward( return attn_output, attn_weights -def bidirectional_mask_function(attention_mask: torch.Tensor | None) -> Callable: - """ - This creates bidirectional attention mask. - """ - - def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - if attention_mask is None: - return torch.ones((), dtype=torch.bool) - return attention_mask[batch_idx, kv_idx].to(torch.bool) - - return inner_mask - - -def sliding_window_bidirectional_mask_function(sliding_window: int) -> Callable: - """ - This creates bidirectional attention mask with sliding window. - """ - - def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - return (q_idx - sliding_window < kv_idx) & (kv_idx < q_idx + sliding_window) - - return inner_mask - - class T5GemmaEncoderLayer(GradientCheckpointingLayer): """Encoder sub-layer.""" @@ -708,20 +686,10 @@ def forward( "config": self.config, "inputs_embeds": inputs_embeds, "attention_mask": attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": position_ids, } self_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(attention_mask), - ), - "sliding_attention": create_sliding_window_causal_mask( - **mask_kwargs, - or_mask_function=sliding_window_bidirectional_mask_function(self.config.sliding_window), - and_mask_function=bidirectional_mask_function(attention_mask), - ), + "full_attention": create_bidirectional_mask(**mask_kwargs), + "sliding_attention": create_bidirectional_sliding_window_mask(**mask_kwargs), } hidden_states = inputs_embeds @@ -825,19 +793,13 @@ def forward( } if not isinstance(cross_attn_mask_mapping := encoder_attention_mask, dict): - mask_kwargs = { - "config": self.config, - "inputs_embeds": encoder_hidden_states, - "attention_mask": encoder_attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": None, - } cross_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(encoder_attention_mask), - ), + "full_attention": create_bidirectional_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=encoder_attention_mask, + encoder_hidden_states=encoder_hidden_states, + ) } hidden_states = inputs_embeds diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index a022d5a9436f..e6ee32fcce75 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -968,19 +968,6 @@ def forward( return outputs -def bidirectional_mask_function(attention_mask: torch.Tensor | None) -> Callable: - """ - This creates bidirectional attention mask. - """ - - def inner_mask(batch_idx: int, head_idx: int, q_idx: int, kv_idx: int) -> bool: - if attention_mask is None: - return torch.ones((), dtype=torch.bool) - return attention_mask[batch_idx, kv_idx].to(torch.bool) - - return inner_mask - - class T5Gemma2Decoder(T5Gemma2PreTrainedModel): config: T5Gemma2DecoderConfig _can_record_outputs = { From 8d9a5d4f2fac88ce3455fdedf7e2581e50541c77 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 11:28:26 +0100 Subject: [PATCH 21/42] oupsi --- .../models/t5gemma/modeling_t5gemma.py | 7 ++++++- .../models/t5gemma/modular_t5gemma.py | 10 ++++++---- .../models/t5gemma2/modeling_t5gemma2.py | 18 ++++++------------ .../models/t5gemma2/modular_t5gemma2.py | 19 ++++++------------- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index 942bc480f988..8e1c5fbd4707 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -29,7 +29,12 @@ from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache from ...generation import GenerationMixin from ...integrations import use_kernel_func_from_hub, use_kernelized_func -from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask +from ...masking_utils import ( + create_bidirectional_mask, + create_bidirectional_sliding_window_mask, + create_causal_mask, + create_sliding_window_causal_mask, +) from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( diff --git a/src/transformers/models/t5gemma/modular_t5gemma.py b/src/transformers/models/t5gemma/modular_t5gemma.py index 10cdf4e9e3dd..3ce46360e346 100644 --- a/src/transformers/models/t5gemma/modular_t5gemma.py +++ b/src/transformers/models/t5gemma/modular_t5gemma.py @@ -22,6 +22,12 @@ from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache from ...configuration_utils import PreTrainedConfig from ...generation import GenerationMixin +from ...masking_utils import ( + create_bidirectional_mask, + create_bidirectional_sliding_window_mask, + create_causal_mask, + create_sliding_window_causal_mask, +) from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( @@ -50,10 +56,6 @@ Gemma2PreTrainedModel, Gemma2RMSNorm, Gemma2RotaryEmbedding, - create_bidirectional_mask, - create_bidirectional_sliding_window_mask, - create_causal_mask, - create_sliding_window_causal_mask, eager_attention_forward, ) diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index e6ee32fcce75..b11ae4ddba6a 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -1052,19 +1052,13 @@ def forward( } if not isinstance(cross_attn_mask_mapping := encoder_attention_mask, dict): - mask_kwargs = { - "config": self.config, - "inputs_embeds": encoder_hidden_states, - "attention_mask": encoder_attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": None, - } cross_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(encoder_attention_mask), - ), + "full_attention": create_bidirectional_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=encoder_attention_mask, + encoder_hidden_states=encoder_hidden_states, + ) } merged_attn_mask_mapping = { diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 3ccdb9d672d3..1e7bd011937d 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -66,7 +66,6 @@ T5GemmaClassificationHead, T5GemmaEncoderLayer, T5GemmaLMHead, - bidirectional_mask_function, ) @@ -1083,19 +1082,13 @@ def forward( } if not isinstance(cross_attn_mask_mapping := encoder_attention_mask, dict): - mask_kwargs = { - "config": self.config, - "inputs_embeds": encoder_hidden_states, - "attention_mask": encoder_attention_mask, - "cache_position": cache_position, - "past_key_values": None, - "position_ids": None, - } cross_attn_mask_mapping = { - "full_attention": create_causal_mask( - **mask_kwargs, - or_mask_function=bidirectional_mask_function(encoder_attention_mask), - ), + "full_attention": create_bidirectional_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=encoder_attention_mask, + encoder_hidden_states=encoder_hidden_states, + ) } merged_attn_mask_mapping = { From 82bb20c2deb846a2134317f6fbe200df913c2a7c Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 11:37:57 +0100 Subject: [PATCH 22/42] oupsi again --- src/transformers/models/t5gemma2/modeling_t5gemma2.py | 8 +++++--- src/transformers/models/t5gemma2/modular_t5gemma2.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index b11ae4ddba6a..4fc7fc0b95dc 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -1035,6 +1035,9 @@ def forward( position_ids = cache_position.unsqueeze(0) if not isinstance(self_attn_mask_mapping := attention_mask, dict): + # this masking function does nothing to masking but forces `allow_is_causal_skip` to be False + # as we always need a mask during decoding for merged attention. + dummy_and_mask_function = lambda *args: torch.tensor(True, dtype=torch.bool) # noqa mask_kwargs = { "config": self.config, "inputs_embeds": inputs_embeds, @@ -1042,10 +1045,8 @@ def forward( "cache_position": cache_position, "past_key_values": past_key_values.self_attention_cache if past_key_values is not None else None, "position_ids": position_ids, + "and_mask_function": dummy_and_mask_function, } - # this masking function did nothing to masking but forces `allow_is_causal_skip` to be False - # as we always need a mask during decoding for merged attention. - mask_kwargs["and_mask_function"] = lambda *args: torch.tensor(True, dtype=torch.bool) self_attn_mask_mapping = { "full_attention": create_causal_mask(**mask_kwargs), "sliding_attention": create_sliding_window_causal_mask(**mask_kwargs), @@ -1058,6 +1059,7 @@ def forward( inputs_embeds=inputs_embeds, attention_mask=encoder_attention_mask, encoder_hidden_states=encoder_hidden_states, + and_mask_function=dummy_and_mask_function, ) } diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 1e7bd011937d..d753d23e3d0b 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -1065,6 +1065,9 @@ def forward( position_ids = cache_position.unsqueeze(0) if not isinstance(self_attn_mask_mapping := attention_mask, dict): + # this masking function does nothing to masking but forces `allow_is_causal_skip` to be False + # as we always need a mask during decoding for merged attention. + dummy_and_mask_function = lambda *args: torch.tensor(True, dtype=torch.bool) # noqa mask_kwargs = { "config": self.config, "inputs_embeds": inputs_embeds, @@ -1072,10 +1075,8 @@ def forward( "cache_position": cache_position, "past_key_values": past_key_values.self_attention_cache if past_key_values is not None else None, "position_ids": position_ids, + "and_mask_function": dummy_and_mask_function, } - # this masking function did nothing to masking but forces `allow_is_causal_skip` to be False - # as we always need a mask during decoding for merged attention. - mask_kwargs["and_mask_function"] = lambda *args: torch.tensor(True, dtype=torch.bool) self_attn_mask_mapping = { "full_attention": create_causal_mask(**mask_kwargs), "sliding_attention": create_sliding_window_causal_mask(**mask_kwargs), @@ -1088,6 +1089,7 @@ def forward( inputs_embeds=inputs_embeds, attention_mask=encoder_attention_mask, encoder_hidden_states=encoder_hidden_states, + and_mask_function=dummy_and_mask_function, ) } From c066e27292ac963f13cd70e8b3bfa17e519db825 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 11:45:43 +0100 Subject: [PATCH 23/42] paddleocr --- src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py | 2 +- src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index c5f3eabcf700..096e8077d916 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -498,7 +498,7 @@ def forward( if position_ids is None: past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.unsqueeze(0) + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) diff --git a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py index 2b777de3a1f9..e2a6f09d79c7 100644 --- a/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modular_paddleocr_vl.py @@ -837,7 +837,7 @@ def forward( if position_ids is None: past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens - position_ids = position_ids.unsqueeze(0) + position_ids = position_ids.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1) elif position_ids.ndim == 2: position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1) From 32fd1ee5fd3329b2bb69e7ce1be824ef39af0d43 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 12:21:16 +0100 Subject: [PATCH 24/42] fix lfms --- src/transformers/models/lfm2/modeling_lfm2.py | 9 ++++++++- src/transformers/models/lfm2/modular_lfm2.py | 9 ++++++++- src/transformers/models/lfm2_moe/modeling_lfm2_moe.py | 9 ++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/lfm2/modeling_lfm2.py b/src/transformers/models/lfm2/modeling_lfm2.py index c2da10284923..adbdf0f78cc7 100644 --- a/src/transformers/models/lfm2/modeling_lfm2.py +++ b/src/transformers/models/lfm2/modeling_lfm2.py @@ -502,7 +502,14 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_length = past_key_values.get_seq_length() + start, end = past_length, past_length + seqlen + # In this case, the cache was already updated and we need to subtract seq_len + if "full_attention" in self.config.layer_types[: self.layer_idx]: + start, end = start - seqlen, end - seqlen + cache_position = torch.arange(start, end, device=x.device) cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2/modular_lfm2.py b/src/transformers/models/lfm2/modular_lfm2.py index 279cc8a55b4c..661aff3b7170 100644 --- a/src/transformers/models/lfm2/modular_lfm2.py +++ b/src/transformers/models/lfm2/modular_lfm2.py @@ -337,7 +337,14 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_length = past_key_values.get_seq_length() + start, end = past_length, past_length + seqlen + # In this case, the cache was already updated and we need to subtract seq_len + if "full_attention" in self.config.layer_types[: self.layer_idx]: + start, end = start - seqlen, end - seqlen + cache_position = torch.arange(start, end, device=x.device) cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py index de29e5055f16..cb2529844370 100644 --- a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py @@ -578,7 +578,14 @@ def slow_forward( if past_key_values is not None and past_key_values.get_seq_length() > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - cache_position = torch.arange(seqlen, device=x.device) + past_key_values.get_seq_length() + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_length = past_key_values.get_seq_length() + start, end = past_length, past_length + seqlen + # In this case, the cache was already updated and we need to subtract seq_len + if "full_attention" in self.config.layer_types[: self.layer_idx]: + start, end = start - seqlen, end - seqlen + cache_position = torch.arange(start, end, device=x.device) cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) From 49cf11180c7b7e5d53f5162ff85b81f5bd713579 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 16:23:12 +0100 Subject: [PATCH 25/42] fix export --- src/transformers/integrations/executorch.py | 53 ++++++++++++++------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/transformers/integrations/executorch.py b/src/transformers/integrations/executorch.py index 15f8bed97c72..4918b609e630 100644 --- a/src/transformers/integrations/executorch.py +++ b/src/transformers/integrations/executorch.py @@ -20,6 +20,7 @@ DynamicSlidingWindowLayer, EncoderDecoderCache, StaticCache, + StaticLayer, ) from ..generation.configuration_utils import GenerationConfig from ..modeling_utils import PreTrainedModel @@ -526,9 +527,13 @@ def __init__( dtype = self.model.dtype # We need this call to initialize all the layers (otherwise it's done lazily, which is not exportable) self.static_cache.early_initialization(batch_size, num_heads, head_dim, dtype, device) - for i in range(len(self.static_cache)): - self.register_buffer(f"key_cache_{i}", self.static_cache.layers[i].keys, persistent=False) - self.register_buffer(f"value_cache_{i}", self.static_cache.layers[i].values, persistent=False) + + # Register cache buffers to make them exportable + for i, layer in enumerate(self.static_cache.layers): + self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) + self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) + if isinstance(layer, StaticLayer): + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward( self, @@ -557,8 +562,12 @@ def forward( The adapter matches the model's forward signature with that in `executorch/extension/llm/runner`, ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box. """ - # Start by resetting static cache - self.static_cache.reset() + # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # already exported) + for layer in self.static_cache.layers: + if isinstance(layer, StaticLayer): + layer.cumulative_length.copy_(cache_position[0:1]) past_key_values = self.static_cache @@ -696,10 +705,12 @@ def __init__( # We need this call to initialize all the layers (otherwise it's done lazily, which is not exportable) self.cache.early_initialization(batch_size, num_heads, head_dim, dtype, device) - # Register all key and value cache tensors as buffers - for i in range(len(self.cache)): - self.register_buffer(f"key_cache_{i}", self.cache.layers[i].keys, persistent=False) - self.register_buffer(f"value_cache_{i}", self.cache.layers[i].values, persistent=False) + # Register cache buffers to make them exportable + for i, layer in enumerate(self.cache.layers): + self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) + self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) + if isinstance(layer, StaticLayer): + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward( self, @@ -718,8 +729,12 @@ def forward( Returns: torch.Tensor: Logits output from the model. """ - # Start by resetting static cache - self.cache.reset() + # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # already exported) + for layer in self.cache.layers: + if isinstance(layer, StaticLayer): + layer.cumulative_length.copy_(cache_position[0:1]) # Forward pass with the model outputs = self.model( @@ -844,13 +859,19 @@ def __init__(self, model, max_static_cache_length, batch_size): register_dynamic_cache_export_support() # Register cache buffers to make them exportable - for i in range(len(self.static_cache)): - self.register_buffer(f"key_cache_{i}", self.static_cache.layers[i].keys, persistent=False) - self.register_buffer(f"value_cache_{i}", self.static_cache.layers[i].values, persistent=False) + for i, layer in enumerate(self.static_cache.layers): + self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) + self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) + if isinstance(layer, StaticLayer): + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward(self, decoder_input_ids, encoder_hidden_states, cache_position): - # Start by resetting static cache - self.cache.reset() + # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # already exported) + for layer in self.static_cache.layers: + if isinstance(layer, StaticLayer): + layer.cumulative_length.copy_(cache_position[0:1]) # Get outputs from decoder outputs = self.decoder( From 3605140fa5c27995bb4a221f0eb43c778902dd5a Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 18:48:20 +0100 Subject: [PATCH 26/42] more export fix --- src/transformers/integrations/executorch.py | 40 +++++++++++++-------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/transformers/integrations/executorch.py b/src/transformers/integrations/executorch.py index 4918b609e630..4eae58649977 100644 --- a/src/transformers/integrations/executorch.py +++ b/src/transformers/integrations/executorch.py @@ -21,6 +21,7 @@ EncoderDecoderCache, StaticCache, StaticLayer, + StaticSlidingWindowLayer, ) from ..generation.configuration_utils import GenerationConfig from ..modeling_utils import PreTrainedModel @@ -522,6 +523,11 @@ def __init__( # Initialize the static cache self.model = model self.static_cache = StaticCache(max_cache_len=max_cache_len, config=config) + # Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by + # simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported + for i, layer in enumerate(self.static_cache.layers): + if isinstance(layer, StaticSlidingWindowLayer): + self.static_cache.layers[i] = StaticLayer(layer.max_cache_len) head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) num_heads = getattr(config, "num_key_value_heads", config.num_attention_heads) dtype = self.model.dtype @@ -532,8 +538,7 @@ def __init__( for i, layer in enumerate(self.static_cache.layers): self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) - if isinstance(layer, StaticLayer): - self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward( self, @@ -563,11 +568,10 @@ def forward( ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box. """ # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, - # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was # already exported) for layer in self.static_cache.layers: - if isinstance(layer, StaticLayer): - layer.cumulative_length.copy_(cache_position[0:1]) + layer.cumulative_length.copy_(cache_position[0:1]) past_key_values = self.static_cache @@ -699,6 +703,11 @@ def __init__( # Initialize the cache self.cache = StaticCache(config=config, max_cache_len=max_cache_len) + # Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by + # simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported + for i, layer in enumerate(self.cache.layers): + if isinstance(layer, StaticSlidingWindowLayer): + self.cache.layers[i] = StaticLayer(layer.max_cache_len) head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads) num_heads = getattr(config, "num_key_value_heads", config.num_attention_heads) dtype = self.model.dtype @@ -709,8 +718,7 @@ def __init__( for i, layer in enumerate(self.cache.layers): self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) - if isinstance(layer, StaticLayer): - self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward( self, @@ -730,11 +738,10 @@ def forward( torch.Tensor: Logits output from the model. """ # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, - # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was # already exported) for layer in self.cache.layers: - if isinstance(layer, StaticLayer): - layer.cumulative_length.copy_(cache_position[0:1]) + layer.cumulative_length.copy_(cache_position[0:1]) # Forward pass with the model outputs = self.model( @@ -851,6 +858,11 @@ def __init__(self, model, max_static_cache_length, batch_size): # Initialize static cache for decoder and DynamicCache for encoder self.static_cache = StaticCache(config=self.config, max_cache_len=max_static_cache_length) + # Since StaticSlidingWindow have dynamic control flow that cannot be avoided, we have to replace them here by + # simple StaticLayer... It means that any generation beyond the window is unfortunately unsupported + for i, layer in enumerate(self.static_cache.layers): + if isinstance(layer, StaticSlidingWindowLayer): + self.static_cache.layers[i] = StaticLayer(layer.max_cache_len) head_dim = getattr(self.config, "head_dim", self.config.hidden_size // self.config.num_attention_heads) num_heads = getattr(self.config, "num_key_value_heads", self.config.num_attention_heads) self.static_cache.early_initialization(batch_size, num_heads, head_dim, torch.float32, model_device) @@ -862,16 +874,14 @@ def __init__(self, model, max_static_cache_length, batch_size): for i, layer in enumerate(self.static_cache.layers): self.register_buffer(f"key_cache_{i}", layer.keys, persistent=False) self.register_buffer(f"value_cache_{i}", layer.values, persistent=False) - if isinstance(layer, StaticLayer): - self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) + self.register_buffer(f"cumulative_length_{i}", layer.cumulative_length, persistent=False) def forward(self, decoder_input_ids, encoder_hidden_states, cache_position): # Start by resetting static cache (it's needed to be able to run several generations with the same exported program, - # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was + # as otherwise it's mutated in-place indefinitely - we cannot call reset in-between the `generate` as the program was # already exported) for layer in self.static_cache.layers: - if isinstance(layer, StaticLayer): - layer.cumulative_length.copy_(cache_position[0:1]) + layer.cumulative_length.copy_(cache_position[0:1]) # Get outputs from decoder outputs = self.decoder( From ae595549b6c4d8ca651c085a2a5784f35d7458f8 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 25 Feb 2026 19:41:02 +0100 Subject: [PATCH 27/42] lfm2 --- src/transformers/models/lfm2/modeling_lfm2.py | 18 +++++++++--------- src/transformers/models/lfm2/modular_lfm2.py | 18 +++++++++--------- .../models/lfm2_moe/modeling_lfm2_moe.py | 18 +++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/transformers/models/lfm2/modeling_lfm2.py b/src/transformers/models/lfm2/modeling_lfm2.py index adbdf0f78cc7..f9ae8deeb865 100644 --- a/src/transformers/models/lfm2/modeling_lfm2.py +++ b/src/transformers/models/lfm2/modeling_lfm2.py @@ -500,16 +500,16 @@ def slow_forward( Bx = B * x - if past_key_values is not None and past_key_values.get_seq_length() > 0: + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + # In this case, the cache was already updated and we need to subtract seq_len to get the correct past length + if "full_attention" in self.config.layer_types[: self.layer_idx]: + past_seen_tokens = past_seen_tokens - seqlen + + if past_key_values is not None and past_seen_tokens > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated - # by the current layer - past_length = past_key_values.get_seq_length() - start, end = past_length, past_length + seqlen - # In this case, the cache was already updated and we need to subtract seq_len - if "full_attention" in self.config.layer_types[: self.layer_idx]: - start, end = start - seqlen, end - seqlen - cache_position = torch.arange(start, end, device=x.device) + cache_position = torch.arange(seqlen, device=conv_state.device) + past_seen_tokens cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2/modular_lfm2.py b/src/transformers/models/lfm2/modular_lfm2.py index 661aff3b7170..eab687df803a 100644 --- a/src/transformers/models/lfm2/modular_lfm2.py +++ b/src/transformers/models/lfm2/modular_lfm2.py @@ -335,16 +335,16 @@ def slow_forward( Bx = B * x - if past_key_values is not None and past_key_values.get_seq_length() > 0: + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + # In this case, the cache was already updated and we need to subtract seq_len to get the correct past length + if "full_attention" in self.config.layer_types[: self.layer_idx]: + past_seen_tokens = past_seen_tokens - seqlen + + if past_key_values is not None and past_seen_tokens > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated - # by the current layer - past_length = past_key_values.get_seq_length() - start, end = past_length, past_length + seqlen - # In this case, the cache was already updated and we need to subtract seq_len - if "full_attention" in self.config.layer_types[: self.layer_idx]: - start, end = start - seqlen, end - seqlen - cache_position = torch.arange(start, end, device=x.device) + cache_position = torch.arange(seqlen, device=conv_state.device) + past_seen_tokens cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) diff --git a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py index cb2529844370..d6b0401e4658 100644 --- a/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py +++ b/src/transformers/models/lfm2_moe/modeling_lfm2_moe.py @@ -576,16 +576,16 @@ def slow_forward( Bx = B * x - if past_key_values is not None and past_key_values.get_seq_length() > 0: + # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated + # by the current layer + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + # In this case, the cache was already updated and we need to subtract seq_len to get the correct past length + if "full_attention" in self.config.layer_types[: self.layer_idx]: + past_seen_tokens = past_seen_tokens - seqlen + + if past_key_values is not None and past_seen_tokens > 0: conv_state = past_key_values.conv_cache[self.layer_idx] - # Note: we may or may not have to substract the current seq_len here as the cache may or may not be already updated - # by the current layer - past_length = past_key_values.get_seq_length() - start, end = past_length, past_length + seqlen - # In this case, the cache was already updated and we need to subtract seq_len - if "full_attention" in self.config.layer_types[: self.layer_idx]: - start, end = start - seqlen, end - seqlen - cache_position = torch.arange(start, end, device=x.device) + cache_position = torch.arange(seqlen, device=conv_state.device) + past_seen_tokens cache_position = cache_position.clamp(0, self.L_cache - 1) conv_state = conv_state.roll(shifts=-1, dims=-1) conv_state[:, :, cache_position] = Bx.to(device=conv_state.device, dtype=conv_state.dtype) From 418a02d109bddc4f91a74ee9a2644eec9c52ec3a Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 13:51:09 +0100 Subject: [PATCH 28/42] fix cuda graph super weird issue --- src/transformers/cache_utils.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 9e379591936c..456822731676 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -396,10 +396,7 @@ def update( kv_length = key_states.shape[-2] cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length - cumulative_length = self.cumulative_length - is_full = cumulative_length >= self.max_cache_len - # Update it now that we saved the value above - self.cumulative_length += kv_length + is_full = self.cumulative_length >= self.max_cache_len if is_full: # In general, we should use a much simpler `cat` here as well, independently of the states size. However, @@ -424,14 +421,14 @@ def update( full_key_states = torch.cat((self.keys[:, :, 1:, :], key_states), dim=-2) full_value_states = torch.cat((self.values[:, :, 1:, :], value_states), dim=-2) # Not yet full, but becoming full on this update - elif cumulative_length + key_states.shape[2] > self.max_cache_len: + elif self.cumulative_length + key_states.shape[2] > self.max_cache_len: # Fast prefill path, no need to cat() in this case, as the cache is currently empty - if cumulative_length == 0: + if self.cumulative_length == 0: full_key_states = key_states full_value_states = value_states else: - full_key_states = torch.cat((self.keys[:, :, :cumulative_length, :], key_states), dim=-2) - full_value_states = torch.cat((self.values[:, :, :cumulative_length, :], value_states), dim=-2) + full_key_states = torch.cat((self.keys[:, :, : self.cumulative_length, :], key_states), dim=-2) + full_value_states = torch.cat((self.values[:, :, : self.cumulative_length, :], value_states), dim=-2) else: try: self.keys.index_copy_(2, cache_position, key_states) @@ -443,6 +440,10 @@ def update( # Very important to return the `self` tensors here, as they have the static dynamo address return self.keys, self.values + # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise + # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) + self.cumulative_length += kv_length + # We only cache the last `sliding_window` tokens self.keys.copy_(full_key_states[:, :, -self.max_cache_len :, :]) self.values.copy_(full_value_states[:, :, -self.max_cache_len :, :]) From ece2c789e4644fe5979571b63cb9d5ae9e5f2776 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 14:02:01 +0100 Subject: [PATCH 29/42] oupsi forgot to add it elsewhere --- src/transformers/cache_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 456822731676..e17c52d6daa7 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -414,6 +414,11 @@ def update( # Copy back into `self` (do not just assign again) in order to keep the static dynamo address self.keys.copy_(new_keys) self.values.copy_(new_values) + + # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise + # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) + self.cumulative_length += kv_length + # Very important to return the `self` tensors here, as they have the static dynamo address return self.keys, self.values # Already full but using more than 1 new token (e.g. prefill caching, chat continuation, etc...) @@ -437,6 +442,10 @@ def update( self.keys[:, :, cache_position] = key_states self.values[:, :, cache_position] = value_states + # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise + # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) + self.cumulative_length += kv_length + # Very important to return the `self` tensors here, as they have the static dynamo address return self.keys, self.values From ca733ab8329789e89fe62109aefa72daea10d6a4 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 15:25:49 +0100 Subject: [PATCH 30/42] fix --- src/transformers/cache_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index e17c52d6daa7..febf2364531c 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -325,9 +325,6 @@ def update( # Create a tensor to slice the static kv at the correct indices kv_length = key_states.shape[-2] cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length - # Note: it is very important to update the cumulative length in-place, as otherwise we cannot use cudagraphs - # with mode="reduce_overhead" (it complains about overwriting the tensor) - self.cumulative_length.add_(kv_length) # Update the cache try: @@ -337,6 +334,12 @@ def update( # Fallback for devices like MPS where index_copy_ might not be supported. self.keys[:, :, cache_position] = key_states self.values[:, :, cache_position] = value_states + + # Note: it is very important to update the cumulative length in-place AND after the cache update, as otherwise + # we cannot use cudagraphs with mode="reduce_overhead" (without the first, it complains about overwriting the tensor. + # without the second, it won't crash but will skip cudagraphs) + self.cumulative_length.add_(kv_length) + return self.keys, self.values def get_mask_sizes(self, query_length: int) -> tuple[int, int]: From 4712b293d6df27e7df3ed267562e0600fc8b14c4 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 18:42:22 +0100 Subject: [PATCH 31/42] fix all finally --- src/transformers/cache_utils.py | 71 +++++++++++++++------------------ 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index febf2364531c..7a0ea48efc17 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -260,7 +260,7 @@ class StaticLayer(CacheLayerMixin): def __init__(self, max_cache_len: int): super().__init__() self.max_cache_len = max_cache_len - # Very important that it's a tensor here, to avoid recompiling when we update it + # Very important that it's a tensor here, to avoid recompiling when we update it and use it to create positions self.cumulative_length = torch.tensor([0], dtype=int) def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tensor) -> None: @@ -292,16 +292,15 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens dtype=self.dtype, device=self.device, ) - # We use this check to easily inherit the method in StaticSlidingWindowLayer without overriding to remove this line - if isinstance(self.cumulative_length, torch.Tensor): - self.cumulative_length = self.cumulative_length.to(self.device) + self.cumulative_length = self.cumulative_length.to(self.device) # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, preventing compiled graph - # breaks when updating the cache. However, it is not supported when tracing the graph, so we skip it in this case. - # As prefill should never be compiled, this is not an issue and it will still be run (except when users compile - # prefill explicitly, but this should be avoided!) + # breaks or cudagraph skips due to inplace mutations when updating the cache. However, it is not supported when + # tracing the graph, so we skip it in this case. As prefill should never be compiled, this is not an issue and it + # will still be run (except when users compile prefill explicitly, but this should be avoided!) if not is_torchdynamo_compiling(): torch._dynamo.mark_static_address(self.keys) torch._dynamo.mark_static_address(self.values) + torch._dynamo.mark_static_address(self.cumulative_length) self.is_initialized = True @@ -325,6 +324,8 @@ def update( # Create a tensor to slice the static kv at the correct indices kv_length = key_states.shape[-2] cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length + # Note that has to be performed in-place, as we have a static address that we need to keep + self.cumulative_length.add_(kv_length) # Update the cache try: @@ -335,11 +336,6 @@ def update( self.keys[:, :, cache_position] = key_states self.values[:, :, cache_position] = value_states - # Note: it is very important to update the cumulative length in-place AND after the cache update, as otherwise - # we cannot use cudagraphs with mode="reduce_overhead" (without the first, it complains about overwriting the tensor. - # without the second, it won't crash but will skip cudagraphs) - self.cumulative_length.add_(kv_length) - return self.keys, self.values def get_mask_sizes(self, query_length: int) -> tuple[int, int]: @@ -375,8 +371,8 @@ class StaticSlidingWindowLayer(StaticLayer): def __init__(self, max_cache_len: int, sliding_window: int): effective_max_cache_len = min(sliding_window, max_cache_len) super().__init__(max_cache_len=effective_max_cache_len) - # Here it cannot be a tensor as otherwise we have data-dependent control flows - overwrite it - self.cumulative_length = 0 + # Here, to avoid data-dependent control flows, we also need to use a python int to keep track of the cumulative length + self.cumulative_length_int = 0 def update( self, key_states: torch.Tensor, value_states: torch.Tensor, *args, **kwargs @@ -395,11 +391,11 @@ def update( if not self.is_initialized: self.lazy_initialization(key_states, value_states) - # Create a tensor to slice the static kv at the correct indices kv_length = key_states.shape[-2] - cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length - - is_full = self.cumulative_length >= self.max_cache_len + current_length = self.cumulative_length_int + is_full = current_length >= self.max_cache_len + # Update it now that we saved the value above + self.cumulative_length_int += kv_length if is_full: # In general, we should use a much simpler `cat` here as well, independently of the states size. However, @@ -418,10 +414,6 @@ def update( self.keys.copy_(new_keys) self.values.copy_(new_values) - # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise - # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) - self.cumulative_length += kv_length - # Very important to return the `self` tensors here, as they have the static dynamo address return self.keys, self.values # Already full but using more than 1 new token (e.g. prefill caching, chat continuation, etc...) @@ -429,15 +421,18 @@ def update( full_key_states = torch.cat((self.keys[:, :, 1:, :], key_states), dim=-2) full_value_states = torch.cat((self.values[:, :, 1:, :], value_states), dim=-2) # Not yet full, but becoming full on this update - elif self.cumulative_length + key_states.shape[2] > self.max_cache_len: + elif current_length + kv_length > self.max_cache_len: # Fast prefill path, no need to cat() in this case, as the cache is currently empty - if self.cumulative_length == 0: + if current_length == 0: full_key_states = key_states full_value_states = value_states else: - full_key_states = torch.cat((self.keys[:, :, : self.cumulative_length, :], key_states), dim=-2) - full_value_states = torch.cat((self.values[:, :, : self.cumulative_length, :], value_states), dim=-2) + full_key_states = torch.cat((self.keys[:, :, :current_length, :], key_states), dim=-2) + full_value_states = torch.cat((self.values[:, :, :current_length, :], value_states), dim=-2) else: + # Note: very important to use the tensor version of the cumulative length here, as otherwise cudagraphs + # (triggered by mode="reduced_overhead") will lead to random crashes, as the int would be overwritten + cache_position = torch.arange(kv_length, device=self.device) + self.cumulative_length try: self.keys.index_copy_(2, cache_position, key_states) self.values.index_copy_(2, cache_position, value_states) @@ -445,17 +440,13 @@ def update( self.keys[:, :, cache_position] = key_states self.values[:, :, cache_position] = value_states - # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise - # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) - self.cumulative_length += kv_length + # Update the tensor version of the length in-place (we don't need to update it if we are already outside + # of this branch, as we don't need the tensor anymore) + self.cumulative_length.add_(kv_length) # Very important to return the `self` tensors here, as they have the static dynamo address return self.keys, self.values - # NOTE: it is very important that this update is happening AFTER we used the variable everywhere else - otherwise - # compiling with cuda graphs will crash (even if we try to save the previous value in another variable) - self.cumulative_length += kv_length - # We only cache the last `sliding_window` tokens self.keys.copy_(full_key_states[:, :, -self.max_cache_len :, :]) self.values.copy_(full_value_states[:, :, -self.max_cache_len :, :]) @@ -465,15 +456,15 @@ def update( def get_mask_sizes(self, query_length: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the attention mask""" sliding_window = self.max_cache_len - is_full = self.cumulative_length >= self.max_cache_len + is_full = self.cumulative_length_int >= self.max_cache_len - kv_offset = max(self.cumulative_length - sliding_window + 1, 0) + kv_offset = max(self.cumulative_length_int - sliding_window + 1, 0) # The cache is already full if is_full: kv_length = sliding_window + query_length - 1 # Not yet full, but becoming full on this update - elif self.cumulative_length + query_length > sliding_window: - kv_length = self.cumulative_length + query_length + elif self.cumulative_length_int + query_length > sliding_window: + kv_length = self.cumulative_length_int + query_length # Here the Cache is still smaller than the local size, but we return the local size as it's static else: kv_length = sliding_window @@ -482,7 +473,11 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: def get_seq_length(self) -> int: """Returns the sequence length of the cached states.""" - return self.cumulative_length + return self.cumulative_length_int + + def reset(self): + super().reset() + self.cumulative_length_int = 0 class QuantizedLayer(DynamicLayer): From 29421b01d69481f1a054ab00503b7c88e3951307 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 18:44:29 +0100 Subject: [PATCH 32/42] style --- src/transformers/cache_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index 7a0ea48efc17..d1011910ed78 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -474,7 +474,7 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: def get_seq_length(self) -> int: """Returns the sequence length of the cached states.""" return self.cumulative_length_int - + def reset(self): super().reset() self.cumulative_length_int = 0 From 0e2b882b5e149530f5bf2944787836a014362b15 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Thu, 26 Feb 2026 19:33:01 +0100 Subject: [PATCH 33/42] fix break --- src/transformers/cache_utils.py | 2 +- src/transformers/masking_utils.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index d1011910ed78..c1ca321ffd9d 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -346,7 +346,7 @@ def get_mask_sizes(self, query_length: int) -> tuple[int, int]: def get_seq_length(self) -> int: """Returns the sequence length of the cached states.""" - return self.cumulative_length.item() + return self.cumulative_length if self.is_initialized else 0 def get_max_cache_shape(self) -> int: """Return the maximum cache shape of the cache""" diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index cd1b26c82d06..db323c64b0eb 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -824,6 +824,9 @@ def _preprocess_mask_arguments( # If using a cache, it can give all information about mask sizes based on seen tokens if past_key_values is not None: q_offset = past_key_values.get_seq_length() + # To avoid graph breaks, StaticLayer return a tensor instead of int -> this has no impact on the ops, but we + # need the correct device + q_offset = q_offset.to(inputs_embeds.device) if isinstance(q_offset, torch.Tensor) else q_offset kv_length, kv_offset = past_key_values.get_mask_sizes(q_length, layer_idx) # Otherwise, we infer based on our input else: From 8e79febe07235490d20441a4dcd08f554df44eba Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 27 Feb 2026 10:53:06 +0100 Subject: [PATCH 34/42] latest model --- .../olmo_hybrid/modeling_olmo_hybrid.py | 34 ++++++------------- .../models/olmo_hybrid/modular_olmo_hybrid.py | 21 +++--------- .../modeling_pp_doclayout_v2.py | 6 ++-- 3 files changed, 17 insertions(+), 44 deletions(-) diff --git a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py index 137435819d01..a666c6dbcdd9 100644 --- a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py +++ b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py @@ -143,14 +143,13 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """ Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. """ kv_offset = 0 - query_length = cache_position.shape[0] past_seen_tokens = self.get_seq_length(layer_idx) kv_length = query_length + past_seen_tokens return kv_length, kv_offset @@ -364,7 +363,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -385,8 +383,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -718,7 +715,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: OlmoHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -840,7 +836,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -851,7 +846,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, ) @@ -883,7 +877,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, output_attentions: bool | None = False, **kwargs: Unpack[TransformersKwargs], @@ -894,7 +887,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) hidden_states = residual + hidden_states @@ -973,7 +965,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -985,23 +976,19 @@ def forward( if use_cache and past_key_values is None: past_key_values = OlmoHybridDynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, input_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds # RoPE or NoPE @@ -1018,7 +1005,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1029,7 +1015,7 @@ def forward( past_key_values=past_key_values, ) - def _update_linear_attn_mask(self, attention_mask, cache_position): + def _update_linear_attn_mask(self, attention_mask, past_key_values): """ NOTE: Left-padding is used for linear attention mask. No need for zeroing states when @@ -1037,7 +1023,9 @@ def _update_linear_attn_mask(self, attention_mask, cache_position): 2. Attending to all inputs """ linear_attn_mask = attention_mask - if cache_position[0] > 0 or (attention_mask is not None and torch.all(attention_mask == 1)): + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( + attention_mask is not None and torch.all(attention_mask == 1) + ): linear_attn_mask = None return linear_attn_mask @@ -1068,7 +1056,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs: Unpack[TransformersKwargs], ) -> CausalLMOutputWithPast: @@ -1096,7 +1083,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/olmo_hybrid/modular_olmo_hybrid.py b/src/transformers/models/olmo_hybrid/modular_olmo_hybrid.py index bf22b56bb128..0926bcfbc4a7 100644 --- a/src/transformers/models/olmo_hybrid/modular_olmo_hybrid.py +++ b/src/transformers/models/olmo_hybrid/modular_olmo_hybrid.py @@ -417,7 +417,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -438,8 +437,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -578,7 +576,6 @@ def forward( self, hidden_states: torch.Tensor, cache_params: OlmoHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -694,7 +691,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, output_attentions: bool | None = False, **kwargs: Unpack[TransformersKwargs], @@ -705,7 +701,6 @@ def forward( hidden_states = self.linear_attn( hidden_states=hidden_states, cache_params=past_key_values, - cache_position=cache_position, attention_mask=attention_mask, ) hidden_states = residual + hidden_states @@ -773,7 +768,6 @@ def forward( past_key_values: Cache | None = None, inputs_embeds: torch.FloatTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> BaseModelOutputWithPast: if (input_ids is None) ^ (inputs_embeds is not None): @@ -785,23 +779,19 @@ def forward( if use_cache and past_key_values is None: past_key_values = OlmoHybridDynamicCache(config=self.config) - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, input_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - linear_attn_mask = self._update_linear_attn_mask(attention_mask, cache_position) + linear_attn_mask = self._update_linear_attn_mask(attention_mask, past_key_values) hidden_states = inputs_embeds # RoPE or NoPE @@ -818,7 +808,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/pp_doclayout_v2/modeling_pp_doclayout_v2.py b/src/transformers/models/pp_doclayout_v2/modeling_pp_doclayout_v2.py index 739564147b76..bab187d3d81e 100644 --- a/src/transformers/models/pp_doclayout_v2/modeling_pp_doclayout_v2.py +++ b/src/transformers/models/pp_doclayout_v2/modeling_pp_doclayout_v2.py @@ -886,12 +886,10 @@ def forward(self, boxes, labels=None, mask=None, **kwargs: Unpack[TransformersKw Bounding box coordinates of the detected layout elements **in [0, 1000] scale**. Format is `[x_min, y_min, x_max, y_max]`. The tensor usually contains sorted valid boxes followed by zero-padding. - labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): The **remapped** class indices for each layout element. These are not necessarily the raw detection class IDs, but indices mapped via `config.class_order` (e.g., mapping text/title/figure to specific reading-order category IDs). - mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Boolean or Binary mask indicating valid detected elements after threshold filtering. - True: Valid layout element. @@ -950,13 +948,13 @@ class PPDocLayoutV2ForObjectDetectionOutput(ModelOutput): r""" logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): Classification logits (including no-object) for all queries. - order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Order logits for all queries. The first dimension of each tensor is the batch size. The second dimension is the number of queries. pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). You can use [`~RTDetrImageProcessor.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits for all queries. The first dimension of each tensor is the batch size. The second dimension is the number of queries. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): From 0d6f071b4297e5f8f092645d26354280a314d30a Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 27 Feb 2026 11:01:16 +0100 Subject: [PATCH 35/42] style --- .../models/pp_doclayout_v2/modular_pp_doclayout_v2.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/pp_doclayout_v2/modular_pp_doclayout_v2.py b/src/transformers/models/pp_doclayout_v2/modular_pp_doclayout_v2.py index bd7fc09e4f66..3896cd77f9d9 100644 --- a/src/transformers/models/pp_doclayout_v2/modular_pp_doclayout_v2.py +++ b/src/transformers/models/pp_doclayout_v2/modular_pp_doclayout_v2.py @@ -840,12 +840,10 @@ def forward(self, boxes, labels=None, mask=None, **kwargs: Unpack[TransformersKw Bounding box coordinates of the detected layout elements **in [0, 1000] scale**. Format is `[x_min, y_min, x_max, y_max]`. The tensor usually contains sorted valid boxes followed by zero-padding. - labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): The **remapped** class indices for each layout element. These are not necessarily the raw detection class IDs, but indices mapped via `config.class_order` (e.g., mapping text/title/figure to specific reading-order category IDs). - mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Boolean or Binary mask indicating valid detected elements after threshold filtering. - True: Valid layout element. @@ -904,13 +902,13 @@ class PPDocLayoutV2ForObjectDetectionOutput(ModelOutput): r""" logits (`torch.FloatTensor` of shape `(batch_size, num_queries, num_classes + 1)`): Classification logits (including no-object) for all queries. - order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): - Order logits for all queries. The first dimension of each tensor is the batch size. The second dimension is the number of queries. pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`): Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding possible padding). You can use [`~RTDetrImageProcessor.post_process_object_detection`] to retrieve the unnormalized (absolute) bounding boxes. + order_logits (`tuple` of `torch.FloatTensor` of shape `(batch_size, num_queries, num_queries)`): + Order logits for all queries. The first dimension of each tensor is the batch size. The second dimension is the number of queries. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`): Sequence of hidden-states at the output of the last layer of the decoder of the model. intermediate_hidden_states (`torch.FloatTensor` of shape `(batch_size, config.decoder_layers, num_queries, hidden_size)`): From c2d8180f862325baceb07b270d8a52b4707f0019 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Fri, 27 Feb 2026 14:11:33 +0100 Subject: [PATCH 36/42] add doc --- src/transformers/masking_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index db323c64b0eb..026c5fd940ca 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -786,6 +786,9 @@ def _preprocess_mask_arguments( layer_idx (`int`, optional): If `past_key_values` is not None, this is the layer index of the cache from which to get the key-value length and offset. Indeed, for hybrid caches, different layers may return different lengths. + encoder_hidden_states (`torch.Tensor`, optional): + The input embeddings of shape (batch_size, kv_length, hidden_dim). If provided, it is used instead of + `inputs_embeds` to infer the kv length. Returns: early_exit (`bool`): From bec65451834b8699229ab213b58977eef0ef6c69 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Mon, 2 Mar 2026 10:28:46 +0100 Subject: [PATCH 37/42] fix audioflamingo --- .../models/audioflamingo3/modeling_audioflamingo3.py | 2 +- .../models/audioflamingo3/modular_audioflamingo3.py | 2 +- src/transformers/models/glmasr/modeling_glmasr.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py index 270ededa0230..c2465febcfb6 100644 --- a/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modeling_audioflamingo3.py @@ -594,7 +594,7 @@ def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if is_first_iteration: + if is_first_iteration or not model_inputs.get("use_cache", False): # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features diff --git a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py index 7e97fd5edcee..e37a60af1107 100644 --- a/src/transformers/models/audioflamingo3/modular_audioflamingo3.py +++ b/src/transformers/models/audioflamingo3/modular_audioflamingo3.py @@ -295,7 +295,7 @@ def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if is_first_iteration: + if is_first_iteration or not model_inputs.get("use_cache", False): # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features diff --git a/src/transformers/models/glmasr/modeling_glmasr.py b/src/transformers/models/glmasr/modeling_glmasr.py index 271f4f8c8bfb..fedcc94f4633 100644 --- a/src/transformers/models/glmasr/modeling_glmasr.py +++ b/src/transformers/models/glmasr/modeling_glmasr.py @@ -502,7 +502,7 @@ def prepare_inputs_for_generation(self, *args, is_first_iteration: bool = False, model_inputs = super().prepare_inputs_for_generation(*args, **kwargs) - if is_first_iteration: + if is_first_iteration or not model_inputs.get("use_cache", False): # input_features should only be passed when we are not in cached decoding stage if input_features is not None: model_inputs["input_features"] = input_features From 3b3f519a634e79a975666d670b4f0b5eca23c2b3 Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Mon, 2 Mar 2026 18:37:06 +0100 Subject: [PATCH 38/42] doc --- src/transformers/cache_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/cache_utils.py b/src/transformers/cache_utils.py index c1ca321ffd9d..7dede60a7b27 100644 --- a/src/transformers/cache_utils.py +++ b/src/transformers/cache_utils.py @@ -293,10 +293,11 @@ def lazy_initialization(self, key_states: torch.Tensor, value_states: torch.Tens device=self.device, ) self.cumulative_length = self.cumulative_length.to(self.device) - # Note: `mark_static_address` is used to tag the cache as a fixed data pointer, preventing compiled graph + # Note: `mark_static_address` is used to tag the tensors as a fixed data pointer, preventing compiled graph # breaks or cudagraph skips due to inplace mutations when updating the cache. However, it is not supported when # tracing the graph, so we skip it in this case. As prefill should never be compiled, this is not an issue and it # will still be run (except when users compile prefill explicitly, but this should be avoided!) + # Without this, we cannot use cudagraphs!! if not is_torchdynamo_compiling(): torch._dynamo.mark_static_address(self.keys) torch._dynamo.mark_static_address(self.values) From 4056eb4c26256bb0f2ad961e0a62bc042605a21f Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Mon, 2 Mar 2026 18:51:53 +0100 Subject: [PATCH 39/42] bc on internal mask functions --- src/transformers/masking_utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index 026c5fd940ca..ab131d4c0460 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -483,6 +483,14 @@ def sdpa_mask( ``` """ + # For BC on `cache_positions` that used to be an arg at the position of `q_length` + if isinstance(q_length, torch.Tensor): + logger.warning_once( + "`cache_position` is deprecated as an arg, and will be removed in Transformers v6. Please use `q_length` and " + "`q_offset` instead, similarly to `kv_length` and `kv_offset`" + ) + q_length, q_offset = q_length.shape[0], q_length[0].to(device) + # Potentially pad the 2D mask padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset) @@ -680,6 +688,14 @@ def flex_attention_mask( device (`torch.device` or `str`, optional): An optional device to create the mask on. """ + # For BC on `cache_positions` that used to be an arg at the position of `q_length` + if isinstance(q_length, torch.Tensor): + logger.warning_once( + "`cache_position` is deprecated as an arg, and will be removed in Transformers v6. Please use `q_length` and " + "`q_offset` instead, similarly to `kv_length` and `kv_offset`" + ) + q_length, q_offset = q_length.shape[0], q_length[0].to(device) + # Potentially add the padding 2D mask if attention_mask is not None: # Older torch (2.5.x) cannot handle sequences not in multiples of 128 (default block size) From 8f6ca3d5d4cf06492940566b491f39fe4d5a65cc Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Mon, 2 Mar 2026 19:16:41 +0100 Subject: [PATCH 40/42] oupsi version --- src/transformers/masking_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index ab131d4c0460..69848473b7ae 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -486,7 +486,7 @@ def sdpa_mask( # For BC on `cache_positions` that used to be an arg at the position of `q_length` if isinstance(q_length, torch.Tensor): logger.warning_once( - "`cache_position` is deprecated as an arg, and will be removed in Transformers v6. Please use `q_length` and " + "`cache_position` is deprecated as an arg, and will be removed in Transformers v5.6. Please use `q_length` and " "`q_offset` instead, similarly to `kv_length` and `kv_offset`" ) q_length, q_offset = q_length.shape[0], q_length[0].to(device) @@ -691,7 +691,7 @@ def flex_attention_mask( # For BC on `cache_positions` that used to be an arg at the position of `q_length` if isinstance(q_length, torch.Tensor): logger.warning_once( - "`cache_position` is deprecated as an arg, and will be removed in Transformers v6. Please use `q_length` and " + "`cache_position` is deprecated as an arg, and will be removed in Transformers v5.6. Please use `q_length` and " "`q_offset` instead, similarly to `kv_length` and `kv_offset`" ) q_length, q_offset = q_length.shape[0], q_length[0].to(device) From 7ac53a9cee7ed5dc2adb1403d0ea66f6c609f24d Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 4 Mar 2026 10:25:09 +0100 Subject: [PATCH 41/42] nemotron --- .../models/nemotron_h/modeling_nemotron_h.py | 42 ++++--------------- .../models/nemotron_h/modular_nemotron_h.py | 26 ++++-------- 2 files changed, 15 insertions(+), 53 deletions(-) diff --git a/src/transformers/models/nemotron_h/modeling_nemotron_h.py b/src/transformers/models/nemotron_h/modeling_nemotron_h.py index feab8075533f..7be887eea05b 100644 --- a/src/transformers/models/nemotron_h/modeling_nemotron_h.py +++ b/src/transformers/models/nemotron_h/modeling_nemotron_h.py @@ -157,25 +157,12 @@ def get_seq_length(self, layer_idx: int | None = 0) -> int: return 0 return self.key_cache[layer_idx].shape[-2] - def get_mask_sizes(self, cache_position: torch.Tensor, layer_idx: int) -> tuple[int, int]: + def get_mask_sizes(self, query_length: int, layer_idx: int) -> tuple[int, int]: """Return the length and offset of the cache, used to generate the mask""" kv_offset = 0 - query_length = cache_position.shape[0] kv_length = self.get_seq_length(layer_idx) + query_length return kv_length, kv_offset - def update_conv_state( - self, layer_idx: int, new_conv_state: torch.Tensor, cache_position: torch.LongTensor - ) -> torch.Tensor: - conv_state = self.conv_states[layer_idx] - cache_position = cache_position.clamp(0, self.conv_kernel_size - 1) - - conv_state = conv_state.roll(shifts=-1, dims=-1) - conv_state[:, :, cache_position] = new_conv_state.to(conv_state.device) - self.conv_states[layer_idx].zero_() - self.conv_states[layer_idx] += conv_state - return self.conv_states[layer_idx] - def reset(self): self.conv_states.zero_() self.ssm_states.zero_() @@ -975,7 +962,6 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: NemotronHHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] @@ -986,9 +972,7 @@ def forward( value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) if past_key_values is not None: - key_states, value_states = past_key_values.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -1046,7 +1030,6 @@ def forward( self, hidden_states, past_key_values: NemotronHHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, use_cache: bool | None = False, @@ -1073,7 +1056,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, user_cache=use_cache, - cache_position=cache_position, **kwargs, ) else: @@ -1184,7 +1166,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: NemotronHHybridDynamicCache | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: @@ -1204,23 +1185,19 @@ def forward( hidden_states = inputs_embeds - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + hidden_states.shape[1], device=hidden_states.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, input_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - mamba_mask = self._update_mamba_mask(attention_mask, cache_position) + mamba_mask = self._update_mamba_mask(attention_mask, past_key_values) # Map block types to their corresponding masks block_type_to_mask = { @@ -1238,7 +1215,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -1252,14 +1228,14 @@ def forward( past_key_values=past_key_values if use_cache else None, ) - def _update_mamba_mask(self, attention_mask, cache_position): + def _update_mamba_mask(self, attention_mask, past_key_values): """ No need for zeroing states when 1. Cached forward 2. Attending to all inputs """ mamba_mask = attention_mask - if (cache_position is not None and cache_position[0] > 0) or ( + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( attention_mask is not None and torch.all(attention_mask == 1) ): mamba_mask = None @@ -1290,7 +1266,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | CausalLMOutputWithPast: @@ -1323,7 +1298,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) diff --git a/src/transformers/models/nemotron_h/modular_nemotron_h.py b/src/transformers/models/nemotron_h/modular_nemotron_h.py index d9b49ef8331e..6720ff681bef 100644 --- a/src/transformers/models/nemotron_h/modular_nemotron_h.py +++ b/src/transformers/models/nemotron_h/modular_nemotron_h.py @@ -19,7 +19,6 @@ import math import torch -import torch.utils.checkpoint from torch import nn from ... import initialization as init @@ -262,10 +261,9 @@ def forward( hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, past_key_values: NemotronHHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor | None]: - return super().forward(hidden_states, attention_mask, past_key_values, cache_position, **kwargs) + return super().forward(hidden_states, attention_mask, past_key_values, **kwargs) MIXER_TYPES = { @@ -304,7 +302,6 @@ def forward( self, hidden_states, past_key_values: NemotronHHybridDynamicCache | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, position_ids: torch.LongTensor | None = None, use_cache: bool | None = False, @@ -331,7 +328,6 @@ def forward( attention_mask=attention_mask, position_ids=position_ids, user_cache=use_cache, - cache_position=cache_position, **kwargs, ) else: @@ -442,7 +438,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: NemotronHHybridDynamicCache | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, attention_mask: torch.Tensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple | BaseModelOutputWithPast: @@ -462,23 +457,19 @@ def forward( hidden_states = inputs_embeds - if cache_position is None: - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + hidden_states.shape[1], device=hidden_states.device - ) if position_ids is None: - position_ids = cache_position.unsqueeze(0) + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + position_ids = torch.arange(hidden_states.shape[1], device=hidden_states.device) + past_seen_tokens + position_ids = position_ids.unsqueeze(0) causal_mask = create_causal_mask( config=self.config, input_embeds=inputs_embeds, attention_mask=attention_mask, - cache_position=cache_position, past_key_values=past_key_values, position_ids=position_ids, ) - mamba_mask = self._update_mamba_mask(attention_mask, cache_position) + mamba_mask = self._update_mamba_mask(attention_mask, past_key_values) # Map block types to their corresponding masks block_type_to_mask = { @@ -496,7 +487,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) @@ -510,14 +500,14 @@ def forward( past_key_values=past_key_values if use_cache else None, ) - def _update_mamba_mask(self, attention_mask, cache_position): + def _update_mamba_mask(self, attention_mask, past_key_values): """ No need for zeroing states when 1. Cached forward 2. Attending to all inputs """ mamba_mask = attention_mask - if (cache_position is not None and cache_position[0] > 0) or ( + if (past_key_values is not None and past_key_values.get_seq_length() > 0) or ( attention_mask is not None and torch.all(attention_mask == 1) ): mamba_mask = None @@ -542,7 +532,6 @@ def forward( inputs_embeds: torch.FloatTensor | None = None, labels: torch.LongTensor | None = None, use_cache: bool | None = None, - cache_position: torch.LongTensor | None = None, logits_to_keep: int | torch.Tensor = 0, **kwargs, ) -> tuple | CausalLMOutputWithPast: @@ -553,7 +542,6 @@ def forward( past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, - cache_position=cache_position, **kwargs, ) From 9030f13de9a3eec9925d06fa56b3f0bb8c4afc5f Mon Sep 17 00:00:00 2001 From: Cyril Vallez Date: Wed, 4 Mar 2026 18:47:13 +0100 Subject: [PATCH 42/42] eurobert --- src/transformers/models/eurobert/modeling_eurobert.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/transformers/models/eurobert/modeling_eurobert.py b/src/transformers/models/eurobert/modeling_eurobert.py index 0023fe48bdbf..7153d5ade132 100644 --- a/src/transformers/models/eurobert/modeling_eurobert.py +++ b/src/transformers/models/eurobert/modeling_eurobert.py @@ -165,7 +165,6 @@ def forward( position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, attention_mask: torch.Tensor | None = None, past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, **kwargs: Unpack[TransformersKwargs], ) -> tuple[torch.Tensor, torch.Tensor]: input_shape = hidden_states.shape[:-1] @@ -179,9 +178,7 @@ def forward( query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) if past_key_values is not None: - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward @@ -237,7 +234,6 @@ def forward( position_ids: torch.LongTensor | None = None, past_key_values: Cache | None = None, use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs: Unpack[TransformersKwargs], ) -> torch.Tensor: @@ -250,7 +246,6 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=use_cache, - cache_position=cache_position, position_embeddings=position_embeddings, **kwargs, )