From d8e4c4f472b139b3012e171a5e50eadf75224936 Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 31 Mar 2026 19:23:54 +0200 Subject: [PATCH 01/33] sdpa x varlen fa --- .../integrations/flash_attention.py | 5 +- src/transformers/masking_utils.py | 1 + .../modeling_flash_attention_utils.py | 149 +++++++++++++----- src/transformers/modeling_utils.py | 40 +++-- src/transformers/utils/__init__.py | 1 + src/transformers/utils/generic.py | 11 +- src/transformers/utils/import_utils.py | 5 + 7 files changed, 149 insertions(+), 63 deletions(-) diff --git a/src/transformers/integrations/flash_attention.py b/src/transformers/integrations/flash_attention.py index 2e878b139be4..301e05f687b7 100644 --- a/src/transformers/integrations/flash_attention.py +++ b/src/transformers/integrations/flash_attention.py @@ -1,13 +1,11 @@ import torch -from ..modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask +from ..modeling_flash_attention_utils import _flash_attention_forward from ..utils import logging logger = logging.get_logger(__name__) -_use_top_left_mask = flash_attn_supports_top_left_mask() - def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtype: """If the query is in float32, return a target dtype compatible with flash attention. Return None otherwise.""" @@ -76,7 +74,6 @@ def flash_attention_forward( softmax_scale=scaling, sliding_window=sliding_window, softcap=softcap, - use_top_left_mask=_use_top_left_mask, target_dtype=target_dtype, attn_implementation=module.config._attn_implementation, layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None, diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index 45e43fdaf3aa..ee403101574f 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -733,6 +733,7 @@ class AttentionMaskInterface(GeneralInterface): "flash_attention_2": flash_attention_mask, "flash_attention_3": flash_attention_mask, "flash_attention_4": flash_attention_mask, + "flash_attention_torch": flash_attention_mask, "flex_attention": flex_attention_mask, } diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 9211ccb19a9e..9c79e3d8be91 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -25,6 +25,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_torch_cuda_available, is_torch_mlu_available, is_torch_npu_available, @@ -106,6 +107,12 @@ def is_flash_attn_available(): "supported_devices": ((is_torch_cuda_available, "cuda"),), "cuda_min_major_version": 9, # Hopper }, + "torch": { + "flash_attn_version": "torch", + "general_availability_check": is_flash_attn_torch_available, + "pkg_availability_check": lambda *args, **kwargs: importlib.util.find_spec("torch") is not None, + "supported_devices": ((is_torch_cuda_available, "cuda"),), # Too uncertain on all features, restricting for now + } } @@ -123,9 +130,17 @@ def is_flash_attn_available(): _hf_api_to_flash_mapping = { "dropout": "dropout_p", "sliding_window": "window_size", + "is_causal": "causal", } # alternative names within the different flash attention APIs, e.g. for attention sinks -_flash_api_alternative_names = {"s_aux": "learnable_sink"} +_flash_api_alternative_names = { + "s_aux": "learnable_sink", + "cu_seqlens_q": "cu_seq_q", + "cu_seqlens_k": "cu_seq_k", + "max_seqlen_q": "max_q", + "max_seqlen_k": "max_k", + "softmax_scale": "scale", +} def _lazy_imports( @@ -150,17 +165,31 @@ def _lazy_imports( is_paged = implementation.startswith("paged|") implementation = implementation.split("|")[1] if is_paged else implementation - if (implementation == "flash_attention_2" and is_fa2) or ( + if implementation in ["sdpa", "flash_attention_torch"]: + from torch.nn.attention import SDPBackend, sdpa_kernel + from torch.nn.attention import varlen as flash_attn_varlen_func + from torch.nn.functional import scaled_dot_product_attention as sdpa + + def flash_attention_wrapper(*args, **kwargs): + with sdpa_kernel(SDPBackend.FLASH_ATTENTION): # Force FA backend to be used + return sdpa(*args, **kwargs) + + flash_attn_func = flash_attention_wrapper + flash_attn_with_kvcache = None # not supported yet + + elif (implementation == "flash_attention_2" and is_fa2) or ( implementation is None and is_fa2 and not is_fa3 and not is_fa4 ): from flash_attn import flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache from flash_attn.bert_padding import pad_input, unpad_input + elif is_torch_npu_available(): # Package `flash-attn` is unavailable on Ascend NPU, which will cause ImportError # Flash-Attention2 related apis for Ascend NPU must be imported from `.integrations.npu_flash_attention` module from .integrations.npu_flash_attention import npu_flash_attn_func as flash_attn_func from .integrations.npu_flash_attention import npu_flash_attn_varlen_func as flash_attn_varlen_func from .integrations.npu_flash_attention import npu_flash_attn_with_kvcache as flash_attn_with_kvcache + else: if implementation == "flash_attention_3" or (implementation is None and is_fa3 and not is_fa4): from flash_attn_interface import flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache @@ -168,6 +197,7 @@ def _lazy_imports( from flash_attn.cute import flash_attn_func, flash_attn_varlen_func flash_attn_with_kvcache = None # not supported yet + # Kernels fallback else: from .integrations.hub_kernels import load_and_register_attn_kernel @@ -565,19 +595,30 @@ class FlashAttentionKwargs(TypedDict, total=False): max_length_k: int | None +def _consolidate_flash_kwarg_alternative_name(kwargs_dict, obj, original_name, supports_mapping): + if obj is None: + return None + + global _flash_api_alternative_names + for name in [original_name, _flash_api_alternative_names.get(original_name, original_name)]: + if supports_mapping[name]: + kwargs_dict[name] = obj + return name + + def _process_flash_attention_kwargs( - query_length: int, - key_length: int, is_causal: bool, dropout: float = 0.0, softmax_scale: float | None = None, sliding_window: int | None = None, - use_top_left_mask: bool = False, softcap: float | None = None, deterministic: bool | None = None, s_aux: torch.Tensor | None = None, max_seqlen_q: int | torch.IntTensor | None = None, max_seqlen_k: int | torch.IntTensor | None = None, + cu_seqlens_q: torch.LongTensor | None = None, + cu_seqlens_k: torch.LongTensor | None = None, + enable_torch_specifics: bool | None = None, supports_mapping: dict[str, bool] | None = None, **kwargs, ): @@ -588,10 +629,6 @@ def _process_flash_attention_kwargs( inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. Args: - query_length (`int`): - Length of the query states - key_length (`int`): - Length of the key states is_causal (`bool`): Whether we perform causal (decoder) attention or full attention. dropout (`float`): @@ -600,8 +637,6 @@ def _process_flash_attention_kwargs( The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. sliding_window (`int`, *optional*): The size of the sliding window, i.e. we look at a max of `sliding_window` tokens back. - use_top_left_mask (`bool`): - Deprecated behavior of older versions of flash attention requiring different masking. softcap (`float`, *optional*): Softcap for the attention logits, used e.g. in gemma2. deterministic (`bool`, *optional*): @@ -612,25 +647,48 @@ def _process_flash_attention_kwargs( The maximum sequence length in the query tensor during a varlen forward. max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): The maximum sequence length in the key/value tensor during a varlen forward. + cu_seqlens_q (`torch.LongTensor]`, *optional*): + The cumulative sequence lengths within the query tensor during a varlen forward. + cu_seqlens_k (`torch.LongTensor]`, *optional*): + The cumulative sequence lengths within the key/value tensor during a varlen forward. + enable_torch_specifics (`bool`, *optional*): + Whether to force torch specific kwargs to be passed (native to the original SDPA backend). Return: flash_kwargs (`dict`): A dict of kwargs that are requested and supported. """ - flash_kwargs = { - "causal": is_causal and not (use_top_left_mask and query_length == 1), - "softmax_scale": softmax_scale, - } + flash_kwargs = {} - if supports_mapping["dropout_p"]: - flash_kwargs["dropout_p"] = dropout + # Every API determined they needed their own convention... + consolidate_flash_kwarg_alternative_name = partial( + _consolidate_flash_kwarg_alternative_name, + kwargs_dict=flash_kwargs, + supports_mapping=supports_mapping, + ) + + for assignable_variable, original_variable_name in zip( + [is_causal, softmax_scale, s_aux, cu_seqlens_q, cu_seqlens_k], + ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"] + ): + consolidate_flash_kwarg_alternative_name( + obj=assignable_variable, + original_name=original_variable_name, + ) - if supports_mapping["window_size"] and sliding_window is not None and key_length > sliding_window: + if supports_mapping["window_size"] and sliding_window is not None: # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left # and the current token for a total size of 5. However, we usually define our window sizes by # their total window size (when causal). Encoder models as of now seldom use SWA and when they # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) + if flash_kwargs.get("causal") is None and flash_kwargs.get("window_size") is None: + # Special case in torch API that utilizes the window to determine underlying causality + flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) + + if supports_mapping["dropout_p"]: + flash_kwargs["dropout_p"] = dropout + if supports_mapping["deterministic"]: flash_kwargs["deterministic"] = ( deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" @@ -639,12 +697,6 @@ def _process_flash_attention_kwargs( if supports_mapping["softcap"] and softcap is not None: flash_kwargs["softcap"] = softcap - if ((legacy_sink_param := supports_mapping["s_aux"]) or supports_mapping["learnable_sink"]) and s_aux is not None: - if legacy_sink_param: - flash_kwargs["s_aux"] = s_aux # e.g. FA3 (vllm) - else: - flash_kwargs["learnable_sink"] = s_aux # FA4 - # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. # @@ -653,17 +705,26 @@ def _process_flash_attention_kwargs( # - Before compiling: `torch._dynamo.config.capture_scalar_outputs = True` # to allow torch compile to handle scalar outputs in those cases. same_max_seqlen = max_seqlen_q is max_seqlen_k # to avoid 2x device syncs - if supports_mapping["max_seqlen_q"] and max_seqlen_q is not None: + + assigned_name_q = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_q, original_name="max_seqlen_q") + if assigned_name_q is not None: if not isinstance(max_seqlen_q, int) and is_tracing(max_seqlen_q): max_seqlen_q = max_seqlen_q.item() - flash_kwargs["max_seqlen_q"] = max_seqlen_q + flash_kwargs[assigned_name_q] = max_seqlen_q - if supports_mapping["max_seqlen_k"] and max_seqlen_k is not None: - if same_max_seqlen and flash_kwargs["max_seqlen_q"] is not None: - max_seqlen_k = flash_kwargs["max_seqlen_q"] - elif not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): - max_seqlen_k = max_seqlen_k.item() - flash_kwargs["max_seqlen_k"] = max_seqlen_k + assigned_name_k = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_k, original_name="max_seqlen_k") + if assigned_name_k is not None: + if same_max_seqlen: + flash_kwargs[assigned_name_k] = flash_kwargs[assigned_name_q] + else: + if not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): + max_seqlen_k = max_seqlen_k.item() + flash_kwargs[assigned_name_k] = max_seqlen_k + + # This is to enable `torch.nn.functional.scaled_dot_product_attention` native FA backend + if flash_kwargs.get("causal") is None and enable_torch_specifics: + flash_kwargs["is_causal"] = is_causal + flash_kwargs["enable_gqa"] = True return flash_kwargs @@ -679,7 +740,6 @@ def _flash_attention_forward( position_ids: torch.Tensor | None = None, softmax_scale: float | None = None, sliding_window: int | None = None, - use_top_left_mask: bool = False, softcap: float | None = None, deterministic: bool | None = None, cu_seq_lens_q: torch.LongTensor | None = None, @@ -721,13 +781,10 @@ def _flash_attention_forward( # Extract the flash attention kwargs that have been requested (and are supported by the implementation) flash_kwargs = partial( process_flash_kwargs_fn, - query_length=query_length, - key_length=key_states.size(1), is_causal=is_causal, dropout=dropout, softmax_scale=softmax_scale, sliding_window=sliding_window, - use_top_left_mask=use_top_left_mask, softcap=softcap, deterministic=deterministic, **kwargs, @@ -760,9 +817,12 @@ def _flash_attention_forward( q, k, v, - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - **flash_kwargs(max_seqlen_q=max_length_q, max_seqlen_k=max_length_k), + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k + ), ) if isinstance(out_unpad, tuple): out_unpad = out_unpad[0] @@ -789,9 +849,12 @@ def _flash_attention_forward( q, k, v, - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - **flash_kwargs(max_seqlen_q=max_length_q, max_seqlen_k=max_length_k), + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k + ), ) if isinstance(out, tuple): out = out[0] @@ -800,7 +863,7 @@ def _flash_attention_forward( # No padding else: - out = flash_fn(query_states, key_states, value_states, **flash_kwargs()) + out = flash_fn(query_states, key_states, value_states, **flash_kwargs(enable_torch_specifics=True)) if isinstance(out, tuple): out = out[0] diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index f50774ef8065..3990c61b8bdc 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -122,9 +122,11 @@ from .utils.hub import DownloadKwargs, create_and_tag_model_card, get_checkpoint_shard_files from .utils.import_utils import ( is_flash_attn_greater_or_equal, + is_flash_attn_torch_available, is_huggingface_hub_greater_or_equal, is_sagemaker_mp_enabled, is_torch_cuda_available, + is_torch_greater_or_equal, is_tracing, ) from .utils.loading_report import LoadStateDictInfo, log_state_dict_report @@ -1556,7 +1558,7 @@ def can_generate(cls) -> bool: def _flash_attn_import_error( self, - flash_attn_version: int, + flash_attn_version: int | str, general_availability_check: Callable, pkg_availability_check: Callable, supported_devices: tuple[tuple[Callable, str]], @@ -1568,7 +1570,7 @@ def _flash_attn_import_error( on why it failed - package import and/or device incompatibility issues. Args: - flash_attn_version (`int`): + flash_attn_version (`Union[int, str]`): The requested version of Flash Attention. general_availability_check (`Callable`): Checks whether our `is_available` function detects the specific FA version. Failing reasons @@ -1595,29 +1597,31 @@ def _flash_attn_import_error( return if not general_availability_check(): - preface = f"FlashAttention{flash_attn_version} has been toggled on, but it cannot be used due to the following error:" + preface = f"Flash Attention {flash_attn_version} has been toggled on, but it cannot be used due to the following error:" # Can the package be seen in the import structure if not pkg_availability_check(): raise ImportError( - f"{preface} the package for FlashAttention{flash_attn_version} doesn't seem to be installed." + f"{preface} the package for Flash Attention {flash_attn_version} doesn't seem to be installed." ) - # Minimum version (FA2 only) + # Minimum version elif flash_attn_version == 2 and not is_flash_attn_greater_or_equal("2.3.3"): - raise ImportError(f"{preface} FlashAttention{flash_attn_version} requires at least version `2.3.3`.") + raise ImportError(f"{preface} Flash Attention {flash_attn_version} requires at least version `2.3.3`.") + elif flash_attn_version == "torch" and not is_torch_greater_or_equal("2.11.0"): + raise ImportError(f"{preface} Flash Attention {flash_attn_version} requires at least version `2.11.0`.") else: # Supported devices availability device_availability_checks, device_names = zip(*supported_devices) if not any(device_availability_check() for device_availability_check in device_availability_checks): raise ImportError( - f"{preface} FlashAttention{flash_attn_version} is not available on CPU. Please make sure you are on any of the supported devices: {device_names}." + f"{preface} Flash Attention {flash_attn_version} is not available on CPU. Please make sure you are on any of the supported devices: {device_names}." ) # Cuda major versions (more recent FA versions are specialized to newer cuda devices) elif cuda_min_major_version is not None and is_torch_cuda_available(): major, _ = torch.cuda.get_device_capability() if major < cuda_min_major_version: raise ImportError( - f"{preface} FlashAttention{flash_attn_version} requires compute capability >= {cuda_min_major_version}, but found {torch.cuda.get_device_capability()} with compute capability {major}.x" + f"{preface} Flash Attention {flash_attn_version} requires compute capability >= {cuda_min_major_version}, but found {torch.cuda.get_device_capability()} with compute capability {major}.x" ) def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool = False) -> bool: @@ -1640,7 +1644,7 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new" ) - if flash_attn_version not in [2, 3, 4]: + if flash_attn_version not in [2, 3, 4, "torch"]: raise ValueError(f"Requested Flash Attention {flash_attn_version} which is not supported.") # Check if we can even use the FA version based on the env of the user @@ -1648,7 +1652,7 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool # Check for attention dropout, which is incompatible with newer FA versions # (many should not really care about dropout as it is not super effective, hence warning for now) - if flash_attn_version > 2: + if flash_attn_version > 2 or flash_attn_version == "torch": if hasattr(self.config, "attention_dropout") and self.config.attention_dropout > 0: logger.warning_once( f"You are attempting to use Flash Attention {flash_attn_version} with dropout. " @@ -1814,6 +1818,10 @@ def _check_and_adjust_attn_implementation( if is_flash_attention_requested(requested_attention_implementation=attn_implementation): # If FA not installed, do not fail but use kernels instead if possible for fa_version in FLASH_ATTENTION_COMPATIBILITY_MATRIX.keys(): + # No fallback for native torch + if fa_version == "torch": + continue + # Check whether we have an original FA requested but not available in the env if requested_original_flash_attn := ( attn_implementation.removeprefix("paged|") == f"flash_attention_{fa_version}" @@ -1867,7 +1875,10 @@ def _check_and_adjust_attn_implementation( ) # preload flash attention here to allow compile with fullgraph - if is_flash_attention_requested(requested_attention_implementation=applicable_attn_implementation): + if is_flash_attention_requested( + requested_attention_implementation=applicable_attn_implementation, + allow_torch=is_flash_attn_torch_available(), + ): lazy_import_flash_attention(applicable_attn_implementation) return applicable_attn_implementation @@ -1906,9 +1917,9 @@ def get_correct_attn_implementation(self, requested_attention: str | None, is_in # Perform relevant checks if is_flash_attention_requested(requested_attention_implementation=applicable_attention) and ( - fa_matched := re.search(r"^flash_attention_(\d)$", applicable_attention) + fa_matched := re.search(r"^flash_attention_(torch|\d)$", applicable_attention) ): - fa_version = int(fa_matched.group(1)) # last digit + fa_version = fa_matched.group(1) if fa_matched.group(1) == "torch" else int(fa_matched.group(1)) # last digit self._flash_attn_can_dispatch(flash_attn_version=fa_version, is_init_check=is_init_check) elif "flex_attention" in applicable_attention: self._flex_attn_can_dispatch(is_init_check) @@ -3773,6 +3784,7 @@ def from_pretrained( - `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)) - `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)) - `"flash_attention_4"` (using [Dao-AILab/flash-attention/flash_attn/cute](https://github.com/Dao-AILab/flash-attention/tree/main/flash_attn/cute)). + - `"flash_attention_torch"` (using a combination of `"sdpa"` and [`varlen`](https://docs.pytorch.org/docs/stable/nn.attention.varlen.html)). By default, if available, SDPA will be used. The default is otherwise the manual `"eager"` implementation. Accept HF kernel references in the form: @@ -4837,11 +4849,13 @@ class AttentionInterface(GeneralInterface): # Class instance object, so that a call to `register` can be reflected into all other files correctly, even if # a new instance is created (in order to locally override a given function) _global_mapping = { + "flash_attention_torch": flash_attention_forward, "flash_attention_4": flash_attention_forward, "flash_attention_3": flash_attention_forward, "flash_attention_2": flash_attention_forward, "flex_attention": flex_attention_forward, "sdpa": sdpa_attention_forward, + "paged|flash_attention_torch": flash_attention_forward, "paged|flash_attention_4": paged_attention_forward, "paged|flash_attention_3": paged_attention_forward, "paged|flash_attention_2": paged_attention_forward, diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 7b8bfb80ec19..b98e9a41019a 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -135,6 +135,7 @@ is_flash_attn_4_available, is_flash_attn_greater_or_equal, is_flash_attn_greater_or_equal_2_10, + is_flash_attn_torch_available, is_flute_available, is_fouroversix_available, is_fp_quant_available, diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index a971a2a61fc0..e2e026c4a4c3 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -248,7 +248,10 @@ def is_mlx_array(x) -> bool: def is_flash_attention_requested( - config=None, requested_attention_implementation: str | None = None, version: int | None = None + config=None, + requested_attention_implementation: str | None = None, + version: int | None = None, + allow_torch: bool | None = None, ) -> bool: """ Checks whether some flavor of flash attention is requested or not. Optionally, checks for a specific version of @@ -260,6 +263,7 @@ def is_flash_attention_requested( The different versions of flash attention are usually - Implementations based on the original flash attention repo: https://github.com/Dao-AILab/flash-attention - Kernels implementations such as: https://huggingface.co/kernels-community/vllm-flash-attn3 + - Or native torch integrations (as of `torch 2.11.0`) if allowed via `allow_torch` """ if config is not None and requested_attention_implementation is not None: raise ValueError( @@ -279,8 +283,9 @@ def is_flash_attention_requested( # If a specific version is requested, look for a pattern of type "flash...{version}" if version is not None: return re.match(r".*flash.*" + str(version), checked_attention_implementation) is not None - # Otherwise, just check "flash" is in the attention implementation - return "flash" in checked_attention_implementation + + # Otherwise, check whether "flash" is in the attention implementation or "sdpa" if `allow_torch` is enabled (flash attention via torch) + return "flash" in checked_attention_implementation or (allow_torch and "sdpa" in checked_attention_implementation) def to_py_obj(obj): diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 9c085ef5bbac..21b19b1b33ed 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -959,6 +959,11 @@ def is_flash_attn_4_available() -> bool: return is_available and is_torch_cuda_available() +@lru_cache +def is_flash_attn_torch_available() -> bool: + return is_torch_greater_or_equal("2.11.0") and is_torch_cuda_available() + + @lru_cache def is_flash_attn_greater_or_equal(library_version: str) -> bool: is_available, flash_attn_version = _is_package_available("flash_attn", return_version=True) From 5c057ff5b036723f00e8ff31dc4eb9118e0ecb09 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 18:30:19 +0000 Subject: [PATCH 02/33] multiple fixes --- .../modeling_flash_attention_utils.py | 44 ++++++++++++------- src/transformers/modeling_utils.py | 11 ++++- src/transformers/utils/generic.py | 8 ++-- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 9c79e3d8be91..62abbe7b214c 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -167,12 +167,13 @@ def _lazy_imports( if implementation in ["sdpa", "flash_attention_torch"]: from torch.nn.attention import SDPBackend, sdpa_kernel - from torch.nn.attention import varlen as flash_attn_varlen_func + from torch.nn.attention.varlen import varlen_attn as flash_attn_varlen_func from torch.nn.functional import scaled_dot_product_attention as sdpa - def flash_attention_wrapper(*args, **kwargs): + # Very limited, not intended for generation but single forward (in encoder-/decoder-only settings) + def flash_attention_wrapper(query, key, value, **kwargs): with sdpa_kernel(SDPBackend.FLASH_ATTENTION): # Force FA backend to be used - return sdpa(*args, **kwargs) + return sdpa(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2), **kwargs) flash_attn_func = flash_attention_wrapper flash_attn_with_kvcache = None # not supported yet @@ -579,9 +580,9 @@ class FlashAttentionKwargs(TypedDict, total=False): Keyword arguments for Flash Attention with Compile. Attributes: - cu_seq_lens_q (`torch.LongTensor`, *optional*) + cu_seq_lens_q (`torch.IntTensor`, *optional*) Gets cumulative sequence length for query state. - cu_seq_lens_k (`torch.LongTensor`, *optional*) + cu_seq_lens_k (`torch.IntTensor`, *optional*) Gets cumulative sequence length for key state. max_length_q (`int`, *optional*): Maximum sequence length for query state. @@ -589,8 +590,8 @@ class FlashAttentionKwargs(TypedDict, total=False): Maximum sequence length for key state. """ - cu_seq_lens_q: torch.LongTensor | None - cu_seq_lens_k: torch.LongTensor | None + cu_seq_lens_q: torch.IntTensor | None + cu_seq_lens_k: torch.IntTensor | None max_length_q: int | None max_length_k: int | None @@ -616,8 +617,8 @@ def _process_flash_attention_kwargs( s_aux: torch.Tensor | None = None, max_seqlen_q: int | torch.IntTensor | None = None, max_seqlen_k: int | torch.IntTensor | None = None, - cu_seqlens_q: torch.LongTensor | None = None, - cu_seqlens_k: torch.LongTensor | None = None, + cu_seqlens_q: torch.IntTensor | None = None, + cu_seqlens_k: torch.IntTensor | None = None, enable_torch_specifics: bool | None = None, supports_mapping: dict[str, bool] | None = None, **kwargs, @@ -647,9 +648,9 @@ def _process_flash_attention_kwargs( The maximum sequence length in the query tensor during a varlen forward. max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): The maximum sequence length in the key/value tensor during a varlen forward. - cu_seqlens_q (`torch.LongTensor]`, *optional*): + cu_seqlens_q (`torch.IntTensor]`, *optional*): The cumulative sequence lengths within the query tensor during a varlen forward. - cu_seqlens_k (`torch.LongTensor]`, *optional*): + cu_seqlens_k (`torch.IntTensor]`, *optional*): The cumulative sequence lengths within the key/value tensor during a varlen forward. enable_torch_specifics (`bool`, *optional*): Whether to force torch specific kwargs to be passed (native to the original SDPA backend). @@ -674,6 +675,7 @@ def _process_flash_attention_kwargs( obj=assignable_variable, original_name=original_variable_name, ) + is_flash_attention_torch = flash_kwargs.get("causal") is None # Due to the unique varlen signature in torch if supports_mapping["window_size"] and sliding_window is not None: # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left @@ -682,9 +684,16 @@ def _process_flash_attention_kwargs( # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) - if flash_kwargs.get("causal") is None and flash_kwargs.get("window_size") is None: - # Special case in torch API that utilizes the window to determine underlying causality + # The torch API inconcistently adopted features for their FA backends + # 1. Varlen can use sliding window but also has to set it to determine causality + # 2. Base FA (non-varlen) has no support for sliding windows (natively) + if is_flash_attention_torch and flash_kwargs.get("window_size") is None: flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) + elif enable_torch_specifics: + raise ValueError( + "Flash Attention torch does not support sliding window through non-varlen paths (yet), e.g. " + "when you don't use an attention mask during inference." + ) if supports_mapping["dropout_p"]: flash_kwargs["dropout_p"] = dropout @@ -722,9 +731,10 @@ def _process_flash_attention_kwargs( flash_kwargs[assigned_name_k] = max_seqlen_k # This is to enable `torch.nn.functional.scaled_dot_product_attention` native FA backend - if flash_kwargs.get("causal") is None and enable_torch_specifics: - flash_kwargs["is_causal"] = is_causal + if is_flash_attention_torch and enable_torch_specifics: flash_kwargs["enable_gqa"] = True + flash_kwargs["is_causal"] = is_causal + flash_kwargs.pop("window_size", None) # unused return flash_kwargs @@ -742,8 +752,8 @@ def _flash_attention_forward( sliding_window: int | None = None, softcap: float | None = None, deterministic: bool | None = None, - cu_seq_lens_q: torch.LongTensor | None = None, - cu_seq_lens_k: torch.LongTensor | None = None, + cu_seq_lens_q: torch.IntTensor | None = None, + cu_seq_lens_k: torch.IntTensor | None = None, max_length_q: int | None = None, max_length_k: int | None = None, target_dtype: torch.dtype | None = None, diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 3990c61b8bdc..2a31aed79efa 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1652,13 +1652,20 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool # Check for attention dropout, which is incompatible with newer FA versions # (many should not really care about dropout as it is not super effective, hence warning for now) - if flash_attn_version > 2 or flash_attn_version == "torch": + if flash_attn_version == "torch" or flash_attn_version > 2: if hasattr(self.config, "attention_dropout") and self.config.attention_dropout > 0: logger.warning_once( f"You are attempting to use Flash Attention {flash_attn_version} with dropout. " "This might lead to unexpected behaviour as this is not supported on recent versions of Flash Attention." ) + if flash_attn_version == "torch": + if getattr(self.config, "sliding_window", None) is not None: + logger.warning_once( + f"You are attempting to use Flash Attention {flash_attn_version} with sliding window. " + "This might lead to unexpected behaviour as this only supported in a limited way with torch natives." + ) + # People often move dtypes after init so we only warn in those cases dtype = self.config.dtype if dtype is None: @@ -1821,7 +1828,7 @@ def _check_and_adjust_attn_implementation( # No fallback for native torch if fa_version == "torch": continue - + # Check whether we have an original FA requested but not available in the env if requested_original_flash_attn := ( attn_implementation.removeprefix("paged|") == f"flash_attention_{fa_version}" diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index e2e026c4a4c3..9076dbcf61a1 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -783,9 +783,9 @@ class TransformersKwargs(TypedDict, total=False): Turn this on to return the intermediary attention scores. output_router_logits (`Optional[bool]`, *optional*): For MoE models, this allows returning the router logits to compute the loss. - cu_seq_lens_q (`torch.LongTensor`, *optional*) + cu_seq_lens_q (`torch.IntTensor`, *optional*) Gets cumulative sequence length for query state. - cu_seq_lens_k (`torch.LongTensor`, *optional*) + cu_seq_lens_k (`torch.IntTensor`, *optional*) Gets cumulative sequence length for key state. max_length_q (`int`, *optional*): Maximum sequence length for query state. @@ -801,8 +801,8 @@ class TransformersKwargs(TypedDict, total=False): output_hidden_states: bool | None output_attentions: bool | None output_router_logits: bool | None - cu_seq_lens_q: torch.LongTensor | None - cu_seq_lens_k: torch.LongTensor | None + cu_seq_lens_q: torch.IntTensor | None + cu_seq_lens_k: torch.IntTensor | None max_length_q: int | None max_length_k: int | None position_ids: torch.LongTensor | None From a763e0291e08fd9e05df0103c1162c5bc16feacb Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 18:44:43 +0000 Subject: [PATCH 03/33] disable generate on non mask path --- src/transformers/generation/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index d07cf05b1ed7..45ade4c71757 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2427,6 +2427,8 @@ def generate( # TODO (joao): generalize this check with other types of inputs if model_input_name == "input_ids" and len(model_kwargs["attention_mask"].shape) > 2: raise ValueError("`attention_mask` passed to `generate` must be 2D.") + if self.config._attn_implementation == "flash_attention_torch" and (model_kwargs["attention_mask"] == 1).all(): + raise ValueError("Flash Attention torch cannot handle generations without valid `attention_mask` with padding.") kwargs_has_position_ids = model_kwargs.get("position_ids", None) is not None accepts_position_ids = "position_ids" in set(inspect.signature(self.forward).parameters.keys()) From ca2ae0c8f32ca970a22e697206816e69a0060ca4 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 19:10:03 +0000 Subject: [PATCH 04/33] circumvent mask creation on packed sequences if varlen available --- src/transformers/masking_utils.py | 21 ++++++++++++++----- src/transformers/modeling_utils.py | 15 ++++++++++++- .../models/llama/modeling_llama.py | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index ee403101574f..c3a71f66eb8f 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -22,7 +22,12 @@ from .utils import is_torch_xpu_available, logging from .utils.deprecation import deprecate_kwarg from .utils.generic import GeneralInterface, is_flash_attention_requested -from .utils.import_utils import is_torch_flex_attn_available, is_torch_greater_or_equal, is_tracing +from .utils.import_utils import ( + is_flash_attn_torch_available, + is_torch_flex_attn_available, + is_torch_greater_or_equal, + is_tracing, +) if is_torch_flex_attn_available(): @@ -834,7 +839,8 @@ def _preprocess_mask_arguments( # full graph dynamo tracing (i.e. torch.export or compile with `fullgraph=True`) will fail on Python<3.11 # with `torch._dynamo.exc.Unsupported: 'inline in skipfiles:Mapping.__contains__ | __contains__, skipped # according trace_rules.lookup SKIP_DIRS'` -- can be removed when we require Python>=3.11 - if config._attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping: + attn_implementation = config._attn_implementation + if attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping: return True, None, None, None, None, None, None # Move the mask to correct device, and potentially switch dtype for efficiency @@ -866,10 +872,15 @@ def _preprocess_mask_arguments( else: kv_length, kv_offset = attention_mask.shape[-1], 0 - # We check the position_ids for potential packed sequence format (only if the 2D attention mask is explicitly None, - # and we don't have past_key_values, i.e. generally a training setup) + # We check the position_ids for potential packed sequence format, if + # 1. There is no 2D attention mask and cache, i.e. generally a training setup + # 2. Or we have some form of flash attention that can handle it natively packed_sequence_mask = None - if position_ids is not None and attention_mask is None and past_key_values is None: + can_skip_packed_mask = is_flash_attention_requested( + requested_attention_implementation=attn_implementation, + allow_torch=is_flash_attn_torch_available(), + ) + if not can_skip_packed_mask and position_ids is not None and attention_mask is None and past_key_values is None: batch_size = inputs_embeds.shape[0] # The position ids are sometimes just unsqueezed, without being expanded if batch_size != position_ids.shape[0]: diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 2a31aed79efa..143f3d6f61b4 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4870,7 +4870,16 @@ class AttentionInterface(GeneralInterface): "paged|eager": eager_paged_attention_forward, } - def get_interface(self, attn_implementation: str, default: Callable) -> Callable: + def get_interface( + self, + attn_implementation: str | None, + default: Callable, + # To determine if we can switch between SDPA and Varlen + attention_mask: torch.Tensor | None, + position_ids: torch.Tensor | None = None, + cu_seq_lens_q: torch.IntTensor | None = None, + **kwargs, + ) -> Callable: """Return the requested `attn_implementation`. Also strictly check its validity, and raise if invalid.""" if attn_implementation is None: logger.warning_once( @@ -4882,6 +4891,10 @@ def get_interface(self, attn_implementation: str, default: Callable) -> Callable raise KeyError( f"`{attn_implementation}` is not a valid attention implementation registered in the `AttentionInterface`" ) + elif attn_implementation == "sdpa" and is_flash_attn_torch_available(): + if attention_mask is None and (position_ids is not None or cu_seq_lens_q is not None): + return flash_attention_forward + return super().get(attn_implementation, default) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 9d659c7c6f08..f8c066713965 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -270,7 +270,7 @@ def forward( key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward + self.config._attn_implementation, eager_attention_forward, attention_mask, **kwargs ) attn_output, attn_weights = attention_interface( From 4470fe841160bd735ee78a5218e478f0d899e29b Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 19:13:26 +0000 Subject: [PATCH 05/33] fix --- src/transformers/modeling_flash_attention_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 62abbe7b214c..0afb58faebdb 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -689,7 +689,7 @@ def _process_flash_attention_kwargs( # 2. Base FA (non-varlen) has no support for sliding windows (natively) if is_flash_attention_torch and flash_kwargs.get("window_size") is None: flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) - elif enable_torch_specifics: + elif is_flash_attention_torch and enable_torch_specifics: raise ValueError( "Flash Attention torch does not support sliding window through non-varlen paths (yet), e.g. " "when you don't use an attention mask during inference." From 2aac2742f973029b8d47742ebfe161e5aa25b662 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 19:18:10 +0000 Subject: [PATCH 06/33] qwen example --- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 6dc8755528d7..ff073ccf0731 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -52,6 +52,7 @@ maybe_autocast, merge_with_config_defaults, ) +from ...utils.import_utils import is_flash_attn_torch_available from ...utils.output_capturing import capture_outputs from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLTextConfig, Qwen2VLVisionConfig @@ -409,10 +410,10 @@ def forward( value_states = value_states.transpose(0, 1).unsqueeze(0) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward + self.config._attn_implementation, eager_attention_forward, None, None, cu_seqlens ) - if is_flash_attention_requested(self.config): + if is_flash_attention_requested(self.config, allow_torch=is_flash_attn_torch_available()): # Flash Attention: Use cu_seqlens for variable length attention max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() attn_output, _ = attention_interface( From dc866c81793de83a873e82bff3f8c6024f8cdbc1 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 19:28:31 +0000 Subject: [PATCH 07/33] fix this --- .../modeling_flash_attention_utils.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 0afb58faebdb..f89d8e1bb988 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -685,15 +685,19 @@ def _process_flash_attention_kwargs( flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) # The torch API inconcistently adopted features for their FA backends - # 1. Varlen can use sliding window but also has to set it to determine causality - # 2. Base FA (non-varlen) has no support for sliding windows (natively) - if is_flash_attention_torch and flash_kwargs.get("window_size") is None: - flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) - elif is_flash_attention_torch and enable_torch_specifics: - raise ValueError( - "Flash Attention torch does not support sliding window through non-varlen paths (yet), e.g. " - "when you don't use an attention mask during inference." - ) + # 1. Base FA (non-varlen) has no support for sliding windows (natively) + # 2. Varlen can use sliding window but also has to set it to determine causality + if is_flash_attention_torch: + if enable_torch_specifics: + raise ValueError( + "Flash Attention torch does not support sliding window through non-varlen paths (yet), e.g. " + "when you don't use an attention mask during inference." + ) + + if flash_kwargs.get("window_size") is None: + flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) + elif is_causal: + flash_kwargs["window_size"] = (flash_kwargs["window_size"][0], 0) if supports_mapping["dropout_p"]: flash_kwargs["dropout_p"] = dropout From ef5d6a602fb9da536d79d25ab89bdc3ff478cb2c Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 20:11:24 +0000 Subject: [PATCH 08/33] style, docs, and remove mask skip on sdpa --- src/transformers/generation/utils.py | 9 ++++-- src/transformers/masking_utils.py | 15 +++------- .../modeling_flash_attention_utils.py | 29 ++++++++++++++----- src/transformers/modeling_utils.py | 11 ++++--- .../models/qwen2_vl/modeling_qwen2_vl.py | 2 +- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 45ade4c71757..2cf819052154 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2427,8 +2427,13 @@ def generate( # TODO (joao): generalize this check with other types of inputs if model_input_name == "input_ids" and len(model_kwargs["attention_mask"].shape) > 2: raise ValueError("`attention_mask` passed to `generate` must be 2D.") - if self.config._attn_implementation == "flash_attention_torch" and (model_kwargs["attention_mask"] == 1).all(): - raise ValueError("Flash Attention torch cannot handle generations without valid `attention_mask` with padding.") + if ( + self.config._attn_implementation == "flash_attention_torch" + and (model_kwargs["attention_mask"] == 1).all() + ): + raise ValueError( + "Flash Attention torch cannot handle generations without valid `attention_mask` with padding." + ) kwargs_has_position_ids = model_kwargs.get("position_ids", None) is not None accepts_position_ids = "position_ids" in set(inspect.signature(self.forward).parameters.keys()) diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index c3a71f66eb8f..e30e7a6125c9 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -23,7 +23,6 @@ from .utils.deprecation import deprecate_kwarg from .utils.generic import GeneralInterface, is_flash_attention_requested from .utils.import_utils import ( - is_flash_attn_torch_available, is_torch_flex_attn_available, is_torch_greater_or_equal, is_tracing, @@ -839,8 +838,7 @@ def _preprocess_mask_arguments( # full graph dynamo tracing (i.e. torch.export or compile with `fullgraph=True`) will fail on Python<3.11 # with `torch._dynamo.exc.Unsupported: 'inline in skipfiles:Mapping.__contains__ | __contains__, skipped # according trace_rules.lookup SKIP_DIRS'` -- can be removed when we require Python>=3.11 - attn_implementation = config._attn_implementation - if attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping: + if config._attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping: return True, None, None, None, None, None, None # Move the mask to correct device, and potentially switch dtype for efficiency @@ -872,15 +870,10 @@ def _preprocess_mask_arguments( else: kv_length, kv_offset = attention_mask.shape[-1], 0 - # We check the position_ids for potential packed sequence format, if - # 1. There is no 2D attention mask and cache, i.e. generally a training setup - # 2. Or we have some form of flash attention that can handle it natively + # We check the position_ids for potential packed sequence format (only if the 2D attention mask is explicitly None, + # and we don't have past_key_values, i.e. generally a training setup) packed_sequence_mask = None - can_skip_packed_mask = is_flash_attention_requested( - requested_attention_implementation=attn_implementation, - allow_torch=is_flash_attn_torch_available(), - ) - if not can_skip_packed_mask and position_ids is not None and attention_mask is None and past_key_values is None: + if position_ids is not None and attention_mask is None and past_key_values is None: batch_size = inputs_embeds.shape[0] # The position ids are sometimes just unsqueezed, without being expanded if batch_size != position_ids.shape[0]: diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index f89d8e1bb988..a114f8194980 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -16,7 +16,7 @@ import os from collections.abc import Callable from functools import partial -from typing import TypedDict +from typing import Any, TypedDict import torch import torch.nn.functional as F @@ -111,8 +111,10 @@ def is_flash_attn_available(): "flash_attn_version": "torch", "general_availability_check": is_flash_attn_torch_available, "pkg_availability_check": lambda *args, **kwargs: importlib.util.find_spec("torch") is not None, - "supported_devices": ((is_torch_cuda_available, "cuda"),), # Too uncertain on all features, restricting for now - } + "supported_devices": ( + (is_torch_cuda_available, "cuda"), + ), # Too uncertain on all features, restricting for now + }, } @@ -596,7 +598,20 @@ class FlashAttentionKwargs(TypedDict, total=False): max_length_k: int | None -def _consolidate_flash_kwarg_alternative_name(kwargs_dict, obj, original_name, supports_mapping): +def _consolidate_flash_kwarg_alternative_name( + kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] +): + """ + TODO + + Args: + kwargs_dict (`dict[str, Any]`): + The current dict of collected kwargs to be passed to the underlying FA function. + obj (`Any`): + The potential object to be passed as kwarg, depending on whether it is supported or not. + original_name (`str`): + The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. + """ if obj is None: return None @@ -669,7 +684,7 @@ def _process_flash_attention_kwargs( for assignable_variable, original_variable_name in zip( [is_causal, softmax_scale, s_aux, cu_seqlens_q, cu_seqlens_k], - ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"] + ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], ): consolidate_flash_kwarg_alternative_name( obj=assignable_variable, @@ -835,7 +850,7 @@ def _flash_attention_forward( cu_seqlens_q=cu_seq_lens_q, cu_seqlens_k=cu_seq_lens_k, max_seqlen_q=max_length_q, - max_seqlen_k=max_length_k + max_seqlen_k=max_length_k, ), ) if isinstance(out_unpad, tuple): @@ -867,7 +882,7 @@ def _flash_attention_forward( cu_seqlens_q=cu_seq_lens_q, cu_seqlens_k=cu_seq_lens_k, max_seqlen_q=max_length_q, - max_seqlen_k=max_length_k + max_seqlen_k=max_length_k, ), ) if isinstance(out, tuple): diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 143f3d6f61b4..b9b21bed558a 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1608,7 +1608,9 @@ def _flash_attn_import_error( elif flash_attn_version == 2 and not is_flash_attn_greater_or_equal("2.3.3"): raise ImportError(f"{preface} Flash Attention {flash_attn_version} requires at least version `2.3.3`.") elif flash_attn_version == "torch" and not is_torch_greater_or_equal("2.11.0"): - raise ImportError(f"{preface} Flash Attention {flash_attn_version} requires at least version `2.11.0`.") + raise ImportError( + f"{preface} Flash Attention {flash_attn_version} requires at least version `2.11.0`." + ) else: # Supported devices availability device_availability_checks, device_names = zip(*supported_devices) @@ -1926,7 +1928,9 @@ def get_correct_attn_implementation(self, requested_attention: str | None, is_in if is_flash_attention_requested(requested_attention_implementation=applicable_attention) and ( fa_matched := re.search(r"^flash_attention_(torch|\d)$", applicable_attention) ): - fa_version = fa_matched.group(1) if fa_matched.group(1) == "torch" else int(fa_matched.group(1)) # last digit + fa_version = ( + fa_matched.group(1) if fa_matched.group(1) == "torch" else int(fa_matched.group(1)) + ) # last digit self._flash_attn_can_dispatch(flash_attn_version=fa_version, is_init_check=is_init_check) elif "flex_attention" in applicable_attention: self._flex_attn_can_dispatch(is_init_check) @@ -4876,7 +4880,6 @@ def get_interface( default: Callable, # To determine if we can switch between SDPA and Varlen attention_mask: torch.Tensor | None, - position_ids: torch.Tensor | None = None, cu_seq_lens_q: torch.IntTensor | None = None, **kwargs, ) -> Callable: @@ -4892,7 +4895,7 @@ def get_interface( f"`{attn_implementation}` is not a valid attention implementation registered in the `AttentionInterface`" ) elif attn_implementation == "sdpa" and is_flash_attn_torch_available(): - if attention_mask is None and (position_ids is not None or cu_seq_lens_q is not None): + if attention_mask is None and cu_seq_lens_q is not None: return flash_attention_forward return super().get(attn_implementation, default) diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index ff073ccf0731..690c983b413a 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -410,7 +410,7 @@ def forward( value_states = value_states.transpose(0, 1).unsqueeze(0) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward, None, None, cu_seqlens + self.config._attn_implementation, eager_attention_forward, None, cu_seqlens ) if is_flash_attention_requested(self.config, allow_torch=is_flash_attn_torch_available()): From 93c239c140bd99e10fc71af46facfd2e5c3aae3f Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 20:12:12 +0000 Subject: [PATCH 09/33] wtf --- src/transformers/modeling_flash_attention_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index a114f8194980..3d136db04848 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -598,11 +598,10 @@ class FlashAttentionKwargs(TypedDict, total=False): max_length_k: int | None -def _consolidate_flash_kwarg_alternative_name( - kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] -): +def _consolidate_flash_kwarg_alternative_name(kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool]): """ - TODO + Consolidates different naming conventions under all FA functions that we support by checking for all alternative names + and whether the FA function supports it. Based on that, it is added to the (existing) kwargs or not. Args: kwargs_dict (`dict[str, Any]`): @@ -611,6 +610,9 @@ def _consolidate_flash_kwarg_alternative_name( The potential object to be passed as kwarg, depending on whether it is supported or not. original_name (`str`): The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. + Return: + name (`str`, *optional*): + The associated name the object was added to the kwargs (if it was added; otherwise None). """ if obj is None: return None From 18af76ff151647ca286a2d4986d71cc80a0b18c8 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 20:12:26 +0000 Subject: [PATCH 10/33] knew it --- src/transformers/modeling_flash_attention_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 3d136db04848..12b22b7be740 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -598,7 +598,9 @@ class FlashAttentionKwargs(TypedDict, total=False): max_length_k: int | None -def _consolidate_flash_kwarg_alternative_name(kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool]): +def _consolidate_flash_kwarg_alternative_name( + kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] +): """ Consolidates different naming conventions under all FA functions that we support by checking for all alternative names and whether the FA function supports it. Based on that, it is added to the (existing) kwargs or not. From 6c09baf221fe17a332a41e8cdc5a73d1ca0a4ce6 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 21:09:38 +0000 Subject: [PATCH 11/33] might revert, test if the varlen fallback is aight --- src/transformers/generation/utils.py | 7 - .../integrations/flash_attention.py | 1 - .../modeling_flash_attention_utils.py | 608 ++++++++++-------- src/transformers/modeling_utils.py | 9 +- 4 files changed, 339 insertions(+), 286 deletions(-) diff --git a/src/transformers/generation/utils.py b/src/transformers/generation/utils.py index 2cf819052154..d07cf05b1ed7 100644 --- a/src/transformers/generation/utils.py +++ b/src/transformers/generation/utils.py @@ -2427,13 +2427,6 @@ def generate( # TODO (joao): generalize this check with other types of inputs if model_input_name == "input_ids" and len(model_kwargs["attention_mask"].shape) > 2: raise ValueError("`attention_mask` passed to `generate` must be 2D.") - if ( - self.config._attn_implementation == "flash_attention_torch" - and (model_kwargs["attention_mask"] == 1).all() - ): - raise ValueError( - "Flash Attention torch cannot handle generations without valid `attention_mask` with padding." - ) kwargs_has_position_ids = model_kwargs.get("position_ids", None) is not None accepts_position_ids = "position_ids" in set(inspect.signature(self.forward).parameters.keys()) diff --git a/src/transformers/integrations/flash_attention.py b/src/transformers/integrations/flash_attention.py index 301e05f687b7..f8a4913839a8 100644 --- a/src/transformers/integrations/flash_attention.py +++ b/src/transformers/integrations/flash_attention.py @@ -76,7 +76,6 @@ def flash_attention_forward( softcap=softcap, target_dtype=target_dtype, attn_implementation=module.config._attn_implementation, - layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None, **kwargs, ) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 12b22b7be740..7f9c538c34a2 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -59,6 +59,27 @@ def is_flash_attn_available(): ) +class FlashAttentionKwargs(TypedDict, total=False): + """ + Keyword arguments for Flash Attention with Compile. + + Attributes: + cu_seq_lens_q (`torch.IntTensor`, *optional*) + Gets cumulative sequence length for query state. + cu_seq_lens_k (`torch.IntTensor`, *optional*) + Gets cumulative sequence length for key state. + max_length_q (`int`, *optional*): + Maximum sequence length for query state. + max_length_k (`int`, *optional*): + Maximum sequence length for key state. + """ + + cu_seq_lens_q: torch.IntTensor | None + cu_seq_lens_k: torch.IntTensor | None + max_length_q: int | None + max_length_k: int | None + + # Mapping from flash attention implementations to their kernel fallback repositories FLASH_ATTN_KERNEL_FALLBACK = { "flash_attention_2": "kernels-community/flash-attn2", @@ -168,16 +189,9 @@ def _lazy_imports( implementation = implementation.split("|")[1] if is_paged else implementation if implementation in ["sdpa", "flash_attention_torch"]: - from torch.nn.attention import SDPBackend, sdpa_kernel from torch.nn.attention.varlen import varlen_attn as flash_attn_varlen_func - from torch.nn.functional import scaled_dot_product_attention as sdpa - - # Very limited, not intended for generation but single forward (in encoder-/decoder-only settings) - def flash_attention_wrapper(query, key, value, **kwargs): - with sdpa_kernel(SDPBackend.FLASH_ATTENTION): # Force FA backend to be used - return sdpa(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2), **kwargs) - flash_attn_func = flash_attention_wrapper + flash_attn_func = None # not supported yet flash_attn_with_kvcache = None # not supported yet elif (implementation == "flash_attention_2" and is_fa2) or ( @@ -221,18 +235,19 @@ def flash_attention_wrapper(query, key, value, **kwargs): f"Could not find the currently requested flash attention implementation at `{implementation}`." "Make sure that you request a valid kernel from the hub, e.g. `kernels-community/flash-attn2`." ) - if flash_attn_func is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` only supports varlen, i.e. " - "it can only be used with continuous batching and does not support the full functionality for " - "the base transformers generation methods." - ) - if flash_attn_with_kvcache is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` does not support block tables, so" - " the full performances of continuous batching will not be achieved, only the varlen path will be " - "used." - ) + + if flash_attn_func is None: + logger.warning( + f"The loaded flash attention implementation at `{implementation}` only supports varlen, i.e. " + "the normal generation pipeline without any padding may produce different results as varlen and base FA " + "have slight numerical differences." + ) + if flash_attn_with_kvcache is None: + logger.warning( + f"The loaded flash attention implementation at `{implementation}` does not support block tables, so" + " the full performances of continuous batching will not be achieved, only the varlen path will be " + "used." + ) return flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache, pad_input, unpad_input @@ -298,6 +313,153 @@ def lazy_import_paged_flash_attention(implementation: str | None, allow_all_kern return flash_attn_varlen_func, flash_attn_with_kvcache_fn +def _consolidate_flash_kwarg_alternative_name( + kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] +): + """ + Consolidates different naming conventions under all FA functions that we support by checking for all alternative names + and whether the FA function supports it. Based on that, it is added to the (existing) kwargs or not. + + Args: + kwargs_dict (`dict[str, Any]`): + The current dict of collected kwargs to be passed to the underlying FA function. + obj (`Any`): + The potential object to be passed as kwarg, depending on whether it is supported or not. + original_name (`str`): + The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. + Return: + name (`str`, *optional*): + The associated name the object was added to the kwargs (if it was added; otherwise None). + """ + if obj is None: + return None + + global _flash_api_alternative_names + for name in [original_name, _flash_api_alternative_names.get(original_name, original_name)]: + if supports_mapping[name]: + kwargs_dict[name] = obj + return name + + +def _process_flash_attention_kwargs( + is_causal: bool, + dropout: float = 0.0, + softmax_scale: float | None = None, + sliding_window: int | None = None, + softcap: float | None = None, + deterministic: bool | None = None, + s_aux: torch.Tensor | None = None, + max_seqlen_q: int | torch.IntTensor | None = None, + max_seqlen_k: int | torch.IntTensor | None = None, + cu_seqlens_q: torch.IntTensor | None = None, + cu_seqlens_k: torch.IntTensor | None = None, + supports_mapping: dict[str, bool] | None = None, + **kwargs, +): + """ + Returns a set of kwargs that are passed down to the according flash attention function based on + requested features and whether it is supported - depends on the version and kernel implementation + which is dynamically configured at `lazy_import_flash_attention`. The (un)supported features can be + inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. + + Args: + is_causal (`bool`): + Whether we perform causal (decoder) attention or full attention. + dropout (`float`): + Attention dropout. + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. + sliding_window (`int`, *optional*): + The size of the sliding window, i.e. we look at a max of `sliding_window` tokens back. + softcap (`float`, *optional*): + Softcap for the attention logits, used e.g. in gemma2. + deterministic (`bool`, *optional*): + Determines if the deterministic option introduced in flash_attn>=2.4.1 is enabled. + s_aux (`torch.Tensor`, *optional*): + Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head. + max_seqlen_q (`Union[int, torch.IntTensor]`, *optional*): + The maximum sequence length in the query tensor during a varlen forward. + max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): + The maximum sequence length in the key/value tensor during a varlen forward. + cu_seqlens_q (`torch.IntTensor]`, *optional*): + The cumulative sequence lengths within the query tensor during a varlen forward. + cu_seqlens_k (`torch.IntTensor]`, *optional*): + The cumulative sequence lengths within the key/value tensor during a varlen forward. + Return: + flash_kwargs (`dict`): + A dict of kwargs that are requested and supported. + """ + flash_kwargs = {} + + # Every API determined they needed their own convention... + consolidate_flash_kwarg_alternative_name = partial( + _consolidate_flash_kwarg_alternative_name, + kwargs_dict=flash_kwargs, + supports_mapping=supports_mapping, + ) + + for assignable_variable, original_variable_name in zip( + [is_causal, softmax_scale, s_aux, cu_seqlens_q, cu_seqlens_k], + ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], + ): + consolidate_flash_kwarg_alternative_name( + obj=assignable_variable, + original_name=original_variable_name, + ) + is_flash_attention_torch = flash_kwargs.get("causal") is None # Due to the unique varlen signature in torch + + if supports_mapping["window_size"] and sliding_window is not None: + # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left + # and the current token for a total size of 5. However, we usually define our window sizes by + # their total window size (when causal). Encoder models as of now seldom use SWA and when they + # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. + flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) + + # Torch varlen can use sliding window but also has to set it to determine causality + if is_flash_attention_torch: + if flash_kwargs.get("window_size") is None: + flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) + elif is_causal: + flash_kwargs["window_size"] = (flash_kwargs["window_size"][0], 0) + + if supports_mapping["dropout_p"]: + flash_kwargs["dropout_p"] = dropout + + if supports_mapping["deterministic"]: + flash_kwargs["deterministic"] = ( + deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + ) + + if supports_mapping["softcap"] and softcap is not None: + flash_kwargs["softcap"] = softcap + + # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` + # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. + # + # You can either set + # - Env: `TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1` + # - Before compiling: `torch._dynamo.config.capture_scalar_outputs = True` + # to allow torch compile to handle scalar outputs in those cases. + same_max_seqlen = max_seqlen_q is max_seqlen_k # to avoid 2x device syncs + + assigned_name_q = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_q, original_name="max_seqlen_q") + if assigned_name_q is not None: + if not isinstance(max_seqlen_q, int) and is_tracing(max_seqlen_q): + max_seqlen_q = max_seqlen_q.item() + flash_kwargs[assigned_name_q] = max_seqlen_q + + assigned_name_k = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_k, original_name="max_seqlen_k") + if assigned_name_k is not None: + if same_max_seqlen: + flash_kwargs[assigned_name_k] = flash_kwargs[assigned_name_q] + else: + if not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): + max_seqlen_k = max_seqlen_k.item() + flash_kwargs[assigned_name_k] = max_seqlen_k + + return flash_kwargs + + def _index_first_axis(tensor, indices): """ A local implementation of the PyTorch indexing operation `tensor[indices]` on the first axis, @@ -467,6 +629,63 @@ def _upad_input( ) +def _flash_attention_mask_varlen( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + unpad_fn: Callable, + pad_fn: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: torch.Tensor, + query_length: int, +): + """ + Manually unpads the tensors based on the attention mask, runs it through the varlen API and sticks + back the to the original sequence lengths by padding with zeros where we manually removed padding values. + """ + q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( + query_states, key_states, value_states, attention_mask, query_length, unpad_fn + ) + + # TODO for now this is required to work with + # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py + if "mps" in str(q.device): + cu_seq_lens_k = cu_seq_lens_k.clone() + + out_unpad = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), + ) + if isinstance(out_unpad, tuple): + out_unpad = out_unpad[0] + + return pad_fn(out_unpad, indices_q, query_states.size(0), query_length) + + +def _is_packed_sequence(position_ids, batch_size): + """ + Check the position ids whether packed sequences are indicated or not + 1. Position ids exist + 2. Flattened sequences only are supported + 3. Compile-friendly `not (torch.diff(position_ids, dim=-1) >= 0).all()`, i.e. we have multiple increasing sequences + """ + if position_ids is None: + return False + + increasing_position_sequences = ( + torch.arange(position_ids.shape[1], device=position_ids.device) + position_ids.min() + ) + return batch_size == 1 and (increasing_position_sequences - position_ids).abs().sum().bool() + + def prepare_fa_kwargs_from_position_ids(position_ids): """ This function returns all the necessary kwargs to call `flash_attn_varlen_func` extracted from position_ids. @@ -543,20 +762,63 @@ def _prepare_from_posids(query, key, value, position_ids): return (query, key, value, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k)) -def _is_packed_sequence(position_ids, batch_size): +def _flash_attention_pure_varlen( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + position_ids: torch.Tensor, + cu_seq_lens_q: torch.IntTensor | None, + cu_seq_lens_k: torch.IntTensor | None, +): """ - Check the position ids whether packed sequences are indicated or not - 1. Position ids exist - 2. Flattened sequences only are supported - 3. Compile-friendly `not (torch.diff(position_ids, dim=-1) >= 0).all()`, i.e. we have multiple increasing sequences + Goes natively through the FA varlen API as the input is already prepared into packed sequences. + The necessary information is already given (`cu_seq_lens`) or we infer it (`position_ids`) ourselves. """ - if position_ids is None: - return False - - increasing_position_sequences = ( - torch.arange(position_ids.shape[1], device=position_ids.device) + position_ids.min() + if cu_seq_lens_q is None or cu_seq_lens_k is None: + q, k, v, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _prepare_from_posids( + query_states, key_states, value_states, position_ids + ) + else: + q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) + k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) + v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) + + # TODO for now this is required to work with + # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py + if "mps" in str(q.device): + cu_seq_lens_k = cu_seq_lens_k.clone() + + out = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), ) - return batch_size == 1 and (increasing_position_sequences - position_ids).abs().sum().bool() + if isinstance(out, tuple): + out = out[0] + + return out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) + + +def _flash_attention_pure( + flash_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, +): + """Native FA API without any padding or masks.""" + out = flash_fn(query_states, key_states, value_states, **flash_kwargs()) + if isinstance(out, tuple): + out = out[0] + return out def fa_peft_integration_check( @@ -577,191 +839,6 @@ def fa_peft_integration_check( return q, k, v -class FlashAttentionKwargs(TypedDict, total=False): - """ - Keyword arguments for Flash Attention with Compile. - - Attributes: - cu_seq_lens_q (`torch.IntTensor`, *optional*) - Gets cumulative sequence length for query state. - cu_seq_lens_k (`torch.IntTensor`, *optional*) - Gets cumulative sequence length for key state. - max_length_q (`int`, *optional*): - Maximum sequence length for query state. - max_length_k (`int`, *optional*): - Maximum sequence length for key state. - """ - - cu_seq_lens_q: torch.IntTensor | None - cu_seq_lens_k: torch.IntTensor | None - max_length_q: int | None - max_length_k: int | None - - -def _consolidate_flash_kwarg_alternative_name( - kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] -): - """ - Consolidates different naming conventions under all FA functions that we support by checking for all alternative names - and whether the FA function supports it. Based on that, it is added to the (existing) kwargs or not. - - Args: - kwargs_dict (`dict[str, Any]`): - The current dict of collected kwargs to be passed to the underlying FA function. - obj (`Any`): - The potential object to be passed as kwarg, depending on whether it is supported or not. - original_name (`str`): - The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. - Return: - name (`str`, *optional*): - The associated name the object was added to the kwargs (if it was added; otherwise None). - """ - if obj is None: - return None - - global _flash_api_alternative_names - for name in [original_name, _flash_api_alternative_names.get(original_name, original_name)]: - if supports_mapping[name]: - kwargs_dict[name] = obj - return name - - -def _process_flash_attention_kwargs( - is_causal: bool, - dropout: float = 0.0, - softmax_scale: float | None = None, - sliding_window: int | None = None, - softcap: float | None = None, - deterministic: bool | None = None, - s_aux: torch.Tensor | None = None, - max_seqlen_q: int | torch.IntTensor | None = None, - max_seqlen_k: int | torch.IntTensor | None = None, - cu_seqlens_q: torch.IntTensor | None = None, - cu_seqlens_k: torch.IntTensor | None = None, - enable_torch_specifics: bool | None = None, - supports_mapping: dict[str, bool] | None = None, - **kwargs, -): - """ - Returns a set of kwargs that are passed down to the according flash attention function based on - requested features and whether it is supported - depends on the version and kernel implementation - which is dynamically configured at `lazy_import_flash_attention`. The (un)supported features can be - inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. - - Args: - is_causal (`bool`): - Whether we perform causal (decoder) attention or full attention. - dropout (`float`): - Attention dropout. - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. - sliding_window (`int`, *optional*): - The size of the sliding window, i.e. we look at a max of `sliding_window` tokens back. - softcap (`float`, *optional*): - Softcap for the attention logits, used e.g. in gemma2. - deterministic (`bool`, *optional*): - Determines if the deterministic option introduced in flash_attn>=2.4.1 is enabled. - s_aux (`torch.Tensor`, *optional*): - Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head. - max_seqlen_q (`Union[int, torch.IntTensor]`, *optional*): - The maximum sequence length in the query tensor during a varlen forward. - max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): - The maximum sequence length in the key/value tensor during a varlen forward. - cu_seqlens_q (`torch.IntTensor]`, *optional*): - The cumulative sequence lengths within the query tensor during a varlen forward. - cu_seqlens_k (`torch.IntTensor]`, *optional*): - The cumulative sequence lengths within the key/value tensor during a varlen forward. - enable_torch_specifics (`bool`, *optional*): - Whether to force torch specific kwargs to be passed (native to the original SDPA backend). - Return: - flash_kwargs (`dict`): - A dict of kwargs that are requested and supported. - """ - flash_kwargs = {} - - # Every API determined they needed their own convention... - consolidate_flash_kwarg_alternative_name = partial( - _consolidate_flash_kwarg_alternative_name, - kwargs_dict=flash_kwargs, - supports_mapping=supports_mapping, - ) - - for assignable_variable, original_variable_name in zip( - [is_causal, softmax_scale, s_aux, cu_seqlens_q, cu_seqlens_k], - ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], - ): - consolidate_flash_kwarg_alternative_name( - obj=assignable_variable, - original_name=original_variable_name, - ) - is_flash_attention_torch = flash_kwargs.get("causal") is None # Due to the unique varlen signature in torch - - if supports_mapping["window_size"] and sliding_window is not None: - # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left - # and the current token for a total size of 5. However, we usually define our window sizes by - # their total window size (when causal). Encoder models as of now seldom use SWA and when they - # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. - flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) - - # The torch API inconcistently adopted features for their FA backends - # 1. Base FA (non-varlen) has no support for sliding windows (natively) - # 2. Varlen can use sliding window but also has to set it to determine causality - if is_flash_attention_torch: - if enable_torch_specifics: - raise ValueError( - "Flash Attention torch does not support sliding window through non-varlen paths (yet), e.g. " - "when you don't use an attention mask during inference." - ) - - if flash_kwargs.get("window_size") is None: - flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) - elif is_causal: - flash_kwargs["window_size"] = (flash_kwargs["window_size"][0], 0) - - if supports_mapping["dropout_p"]: - flash_kwargs["dropout_p"] = dropout - - if supports_mapping["deterministic"]: - flash_kwargs["deterministic"] = ( - deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" - ) - - if supports_mapping["softcap"] and softcap is not None: - flash_kwargs["softcap"] = softcap - - # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` - # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. - # - # You can either set - # - Env: `TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1` - # - Before compiling: `torch._dynamo.config.capture_scalar_outputs = True` - # to allow torch compile to handle scalar outputs in those cases. - same_max_seqlen = max_seqlen_q is max_seqlen_k # to avoid 2x device syncs - - assigned_name_q = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_q, original_name="max_seqlen_q") - if assigned_name_q is not None: - if not isinstance(max_seqlen_q, int) and is_tracing(max_seqlen_q): - max_seqlen_q = max_seqlen_q.item() - flash_kwargs[assigned_name_q] = max_seqlen_q - - assigned_name_k = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_k, original_name="max_seqlen_k") - if assigned_name_k is not None: - if same_max_seqlen: - flash_kwargs[assigned_name_k] = flash_kwargs[assigned_name_q] - else: - if not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): - max_seqlen_k = max_seqlen_k.item() - flash_kwargs[assigned_name_k] = max_seqlen_k - - # This is to enable `torch.nn.functional.scaled_dot_product_attention` native FA backend - if is_flash_attention_torch and enable_torch_specifics: - flash_kwargs["enable_gqa"] = True - flash_kwargs["is_causal"] = is_causal - flash_kwargs.pop("window_size", None) # unused - - return flash_kwargs - - def _flash_attention_forward( query_states: torch.Tensor, key_states: torch.Tensor, @@ -823,6 +900,8 @@ def _flash_attention_forward( **kwargs, ) + # Determine which FA function to call + # We will use `flash_varlen_fn` to prevent cross-example attention and also allow padding free approach under two cases: # Case 1. If position ids is provided and the position ids indicate packed sequences, see `_is_packed_sequence`. # Case 2. Some models pass directly pre-computed `cu_seqlens` so we don't need to infer it from position ids. It is safe to @@ -837,67 +916,56 @@ def _flash_attention_forward( # Contains at least one padding token in the sequence if attention_mask is not None: - q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( - query_states, key_states, value_states, attention_mask, query_length, unpad_fn - ) - - # TODO for now this is required to work with - # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py - if "mps" in str(q.device): - cu_seq_lens_k = cu_seq_lens_k.clone() - - out_unpad = flash_varlen_fn( - q, - k, - v, - **flash_kwargs( - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - max_seqlen_q=max_length_q, - max_seqlen_k=max_length_k, - ), + out = _flash_attention_mask_varlen( + flash_varlen_fn=flash_varlen_fn, + flash_kwargs=flash_kwargs, + unpad_fn=unpad_fn, + pad_fn=pad_fn, + query_states=query_states, + key_states=key_states, + value_states=value_states, + attention_mask=attention_mask, + query_length=query_length, ) - if isinstance(out_unpad, tuple): - out_unpad = out_unpad[0] - - out = pad_fn(out_unpad, indices_q, query_states.size(0), query_length) # Padding free, i.e. sequences flattened into one total sequence elif is_fa_with_varlen_kwargs or is_fa_with_position_ids: - if cu_seq_lens_q is None or cu_seq_lens_k is None: - q, k, v, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _prepare_from_posids( - query_states, key_states, value_states, position_ids - ) - else: - q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) - k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) - v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) - - # TODO for now this is required to work with - # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py - if "mps" in str(q.device): - cu_seq_lens_k = cu_seq_lens_k.clone() - - out = flash_varlen_fn( - q, - k, - v, - **flash_kwargs( - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - max_seqlen_q=max_length_q, - max_seqlen_k=max_length_k, - ), + out = _flash_attention_pure_varlen( + flash_varlen_fn=flash_varlen_fn, + flash_kwargs=flash_kwargs, + query_states=query_states, + key_states=key_states, + value_states=value_states, + position_ids=position_ids, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, ) - if isinstance(out, tuple): - out = out[0] - - out = out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) # No padding else: - out = flash_fn(query_states, key_states, value_states, **flash_kwargs(enable_torch_specifics=True)) - if isinstance(out, tuple): - out = out[0] + # Case 1: No native base FA, we fallback to simulating varlen (but with no padding) - less numerically equivalent + if flash_fn is None: + attention_mask = torch.ones(size=(key_states.shape[:2]), device=key_states.device, dtype=torch.bool) + + out = _flash_attention_mask_varlen( + flash_varlen_fn=flash_varlen_fn, + flash_kwargs=flash_kwargs, + unpad_fn=unpad_fn, + pad_fn=pad_fn, + query_states=query_states, + key_states=key_states, + value_states=value_states, + attention_mask=attention_mask, + query_length=query_length, + ) + # Case 2: Normal base FA as intended + else: + out = _flash_attention_pure( + flash_fn=flash_fn, + flash_kwargs=flash_kwargs, + query_states=query_states, + key_states=key_states, + value_states=value_states, + ) return out diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index b9b21bed558a..3e5c315a1114 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1661,13 +1661,6 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool "This might lead to unexpected behaviour as this is not supported on recent versions of Flash Attention." ) - if flash_attn_version == "torch": - if getattr(self.config, "sliding_window", None) is not None: - logger.warning_once( - f"You are attempting to use Flash Attention {flash_attn_version} with sliding window. " - "This might lead to unexpected behaviour as this only supported in a limited way with torch natives." - ) - # People often move dtypes after init so we only warn in those cases dtype = self.config.dtype if dtype is None: @@ -4879,7 +4872,7 @@ def get_interface( attn_implementation: str | None, default: Callable, # To determine if we can switch between SDPA and Varlen - attention_mask: torch.Tensor | None, + attention_mask: torch.Tensor | None = None, cu_seq_lens_q: torch.IntTensor | None = None, **kwargs, ) -> Callable: From a28a9bf678c689b05f28a05a1c261aede7c37ade Mon Sep 17 00:00:00 2001 From: Vasqu Date: Tue, 31 Mar 2026 21:26:43 +0000 Subject: [PATCH 12/33] fix --- src/transformers/modeling_utils.py | 13 +------------ src/transformers/models/llama/modeling_llama.py | 2 +- .../models/qwen2_vl/modeling_qwen2_vl.py | 6 ++++-- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 3e5c315a1114..1e77c8272cc3 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4867,15 +4867,7 @@ class AttentionInterface(GeneralInterface): "paged|eager": eager_paged_attention_forward, } - def get_interface( - self, - attn_implementation: str | None, - default: Callable, - # To determine if we can switch between SDPA and Varlen - attention_mask: torch.Tensor | None = None, - cu_seq_lens_q: torch.IntTensor | None = None, - **kwargs, - ) -> Callable: + def get_interface(self, attn_implementation: str | None, default: Callable) -> Callable: """Return the requested `attn_implementation`. Also strictly check its validity, and raise if invalid.""" if attn_implementation is None: logger.warning_once( @@ -4887,9 +4879,6 @@ def get_interface( raise KeyError( f"`{attn_implementation}` is not a valid attention implementation registered in the `AttentionInterface`" ) - elif attn_implementation == "sdpa" and is_flash_attn_torch_available(): - if attention_mask is None and cu_seq_lens_q is not None: - return flash_attention_forward return super().get(attn_implementation, default) diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index f8c066713965..9d659c7c6f08 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -270,7 +270,7 @@ def forward( key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward, attention_mask, **kwargs + self.config._attn_implementation, eager_attention_forward ) attn_output, attn_weights = attention_interface( diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 690c983b413a..528260c5a3f5 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -33,6 +33,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub +from ...integrations.flash_attention import flash_attention_forward from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer @@ -410,13 +411,14 @@ def forward( value_states = value_states.transpose(0, 1).unsqueeze(0) attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( - self.config._attn_implementation, eager_attention_forward, None, cu_seqlens + self.config._attn_implementation, eager_attention_forward ) if is_flash_attention_requested(self.config, allow_torch=is_flash_attn_torch_available()): # Flash Attention: Use cu_seqlens for variable length attention max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - attn_output, _ = attention_interface( + # Force FA interface to call underlying SDPA's registered varlen + attn_output, _ = flash_attention_forward( self, query_states, key_states, From cabd09b1575faceabfd4af00bd950dc2fdb38163 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 12:43:07 +0000 Subject: [PATCH 13/33] deprecate top left mask (was already done somewhat before as its unused in all but maybe npu), change error + warning logic, some simplifications --- .../integrations/npu_flash_attention.py | 17 ++-- .../modeling_flash_attention_utils.py | 84 ++++++++++--------- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/src/transformers/integrations/npu_flash_attention.py b/src/transformers/integrations/npu_flash_attention.py index 2af3ba84e012..061b2fcecd25 100644 --- a/src/transformers/integrations/npu_flash_attention.py +++ b/src/transformers/integrations/npu_flash_attention.py @@ -12,6 +12,7 @@ import math import os +import warnings import torch @@ -23,15 +24,14 @@ # FlashAttention2 is supported on Ascend NPU with down-right aligned causal mask by default. -# Set environment variable `NPU_FA2_SPARSE_MODE` to 2 when using top-left aligned causal mask. -TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2 +# It can set an environment variable `NPU_FA2_SPARSE_MODE` to control this behavior. +TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2 # Deprecated DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE = 3 SPARSE_MODE = int(os.getenv("NPU_FA2_SPARSE_MODE", default=DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE)) -if SPARSE_MODE not in [TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE, DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE]: +if SPARSE_MODE != DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE: raise ValueError( - "Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 2 (top-left aligned causal mask) " - "or 3 (down-right aligned causal mask)." + "Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 3 (down-right aligned causal mask)." ) ATTN_MASK_NPU_CACHE = {} @@ -45,7 +45,12 @@ def get_attn_mask_npu(device): def is_npu_fa2_top_left_aligned_causal_mask(): - return SPARSE_MODE == TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE if is_torch_npu_available() else False + warnings.warn( + "`is_npu_fa2_top_left_aligned_causal_mask` is deprecated and will be removed in v5.8. " + "This is no longer needed as the minimum required FA version is no longer affected by this.", + FutureWarning, + ) + return False def npu_flash_attn_func( diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 7f9c538c34a2..4c81decd351a 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -14,6 +14,7 @@ import importlib import inspect import os +import warnings from collections.abc import Callable from functools import partial from typing import Any, TypedDict @@ -38,20 +39,20 @@ logger = logging.get_logger(__name__) -# TODO Deprecate when all models have the attention interface def flash_attn_supports_top_left_mask(): - if is_flash_attn_2_available() or is_flash_attn_3_available() or is_flash_attn_4_available(): - return False - - from .integrations.npu_flash_attention import is_npu_fa2_top_left_aligned_causal_mask - - return is_npu_fa2_top_left_aligned_causal_mask() + warnings.warn( + "`flash_attn_supports_top_left_mask` is deprecated and will be removed in v5.8. " + "This is no longer needed as the minimum required FA version is no longer affected by this.", + FutureWarning, + ) + return False # TODO Deprecate when all models have the attention interface def is_flash_attn_available(): return ( - is_flash_attn_4_available() + is_flash_attn_torch_available() + or is_flash_attn_4_available() or is_flash_attn_3_available() or is_flash_attn_2_available() or is_torch_npu_available() @@ -236,19 +237,6 @@ def _lazy_imports( "Make sure that you request a valid kernel from the hub, e.g. `kernels-community/flash-attn2`." ) - if flash_attn_func is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` only supports varlen, i.e. " - "the normal generation pipeline without any padding may produce different results as varlen and base FA " - "have slight numerical differences." - ) - if flash_attn_with_kvcache is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` does not support block tables, so" - " the full performances of continuous batching will not be achieved, only the varlen path will be " - "used." - ) - return flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache, pad_input, unpad_input @@ -318,7 +306,7 @@ def _consolidate_flash_kwarg_alternative_name( ): """ Consolidates different naming conventions under all FA functions that we support by checking for all alternative names - and whether the FA function supports it. Based on that, it is added to the (existing) kwargs or not. + and whether the FA function supports it. Based on that, it is added to the (existing) kwargs in-place or not. Args: kwargs_dict (`dict[str, Any]`): @@ -340,6 +328,24 @@ def _consolidate_flash_kwarg_alternative_name( kwargs_dict[name] = obj return name + # Torch in specific does not set it, ignore + if original_name != "causal": + # 2 variables with less impact, warn if possible + if original_name in ["dropout_p", "deterministic"] and not is_tracing(obj): + # Check if it goes against defaults + if (original_name == "dropout_p" and obj != 0.0) or (original_name == "deterministic" and obj is True): + logger.warning_once( + f"We detected the usage of {original_name} within Flash Attention. It is not supported in this version " + "of Flash Attention. This will be ignored but may lead to unintended behavior." + ) + # Core features that would highly influence the output + else: + raise ValueError( + f"Tried to assign {original_name}={obj} (or alternative names) to the underlying flash attention function. " + "But couldn't find it in its signature. Please make sure to disable this feature (e.g. dropout) or use a different " + "Flash Attention version (e.g. FA4 for attentions sinks in GPT Oss) that supports it." + ) + def _process_flash_attention_kwargs( is_causal: bool, @@ -398,15 +404,15 @@ def _process_flash_attention_kwargs( supports_mapping=supports_mapping, ) + deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" for assignable_variable, original_variable_name in zip( - [is_causal, softmax_scale, s_aux, cu_seqlens_q, cu_seqlens_k], - ["causal", "softmax_scale", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], + [is_causal, dropout, softmax_scale, softcap, deterministic, s_aux, cu_seqlens_q, cu_seqlens_k], + ["causal", "dropout_p", "softmax_scale", "softcap", "deterministic", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], ): consolidate_flash_kwarg_alternative_name( obj=assignable_variable, original_name=original_variable_name, ) - is_flash_attention_torch = flash_kwargs.get("causal") is None # Due to the unique varlen signature in torch if supports_mapping["window_size"] and sliding_window is not None: # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left @@ -416,23 +422,12 @@ def _process_flash_attention_kwargs( flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) # Torch varlen can use sliding window but also has to set it to determine causality - if is_flash_attention_torch: + if flash_kwargs.get("causal") is None: if flash_kwargs.get("window_size") is None: flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) elif is_causal: flash_kwargs["window_size"] = (flash_kwargs["window_size"][0], 0) - if supports_mapping["dropout_p"]: - flash_kwargs["dropout_p"] = dropout - - if supports_mapping["deterministic"]: - flash_kwargs["deterministic"] = ( - deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" - ) - - if supports_mapping["softcap"] and softcap is not None: - flash_kwargs["softcap"] = softcap - # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. # @@ -637,13 +632,17 @@ def _flash_attention_mask_varlen( query_states: torch.Tensor, key_states: torch.Tensor, value_states: torch.Tensor, - attention_mask: torch.Tensor, + attention_mask: torch.Tensor | None, query_length: int, ): """ Manually unpads the tensors based on the attention mask, runs it through the varlen API and sticks back the to the original sequence lengths by padding with zeros where we manually removed padding values. """ + # Default create an all 1 (True) mask if nothing is given + if attention_mask is None: + attention_mask = torch.ones(size=(key_states.shape[:2]), device=key_states.device, dtype=torch.bool) + q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( query_states, key_states, value_states, attention_mask, query_length, unpad_fn ) @@ -943,9 +942,14 @@ def _flash_attention_forward( # No padding else: - # Case 1: No native base FA, we fallback to simulating varlen (but with no padding) - less numerically equivalent + # Case 1: No native base FA, we fallback to simulating varlen (but with no padding) if flash_fn is None: - attention_mask = torch.ones(size=(key_states.shape[:2]), device=key_states.device, dtype=torch.bool) + if not is_tracing(query_states): + logger.warning_once( + "We detected that your current underlying Flash Attention implementation does not implement a simple base" + "Flash Attention function (non-varlen). This can lead to slight inefficiencies (generation speed) and " + "changes in generation." + ) out = _flash_attention_mask_varlen( flash_varlen_fn=flash_varlen_fn, From 987c26980314315ae3731775aaad1f85f691f6fd Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 12:51:33 +0000 Subject: [PATCH 14/33] oops --- src/transformers/modeling_flash_attention_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 4c81decd351a..667f76a62ea6 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -346,6 +346,8 @@ def _consolidate_flash_kwarg_alternative_name( "Flash Attention version (e.g. FA4 for attentions sinks in GPT Oss) that supports it." ) + return None + def _process_flash_attention_kwargs( is_causal: bool, From 5fb4582fbfac0dfaf097e25fc8728c45b1077eb5 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 13:01:08 +0000 Subject: [PATCH 15/33] make alt names list of str, small nit --- .../modeling_flash_attention_utils.py | 19 ++++++++++--------- src/transformers/modeling_utils.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 667f76a62ea6..02ea98670b16 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -158,12 +158,12 @@ class FlashAttentionKwargs(TypedDict, total=False): } # alternative names within the different flash attention APIs, e.g. for attention sinks _flash_api_alternative_names = { - "s_aux": "learnable_sink", - "cu_seqlens_q": "cu_seq_q", - "cu_seqlens_k": "cu_seq_k", - "max_seqlen_q": "max_q", - "max_seqlen_k": "max_k", - "softmax_scale": "scale", + "s_aux": ["learnable_sink"], + "cu_seqlens_q": ["cu_seq_q"], + "cu_seqlens_k": ["cu_seq_k"], + "max_seqlen_q": ["max_q"], + "max_seqlen_k": ["max_k"], + "softmax_scale": ["scale"], } @@ -258,8 +258,9 @@ def _lazy_define_process_function(flash_function): fa_param = _hf_api_to_flash_mapping.get(param, param) supports_mapping[fa_param] = fa_param in flash_parameters - if (fa_alternative_name := _flash_api_alternative_names.get(param, param)) != fa_param: - supports_mapping[fa_alternative_name] = fa_alternative_name in flash_parameters + if (fa_alternative_name := _flash_api_alternative_names.get(param, fa_param)) != fa_param: + for alternative_name in fa_alternative_name: + supports_mapping[alternative_name] = alternative_name in flash_parameters return partial(_process_flash_attention_kwargs, supports_mapping=supports_mapping) @@ -323,7 +324,7 @@ def _consolidate_flash_kwarg_alternative_name( return None global _flash_api_alternative_names - for name in [original_name, _flash_api_alternative_names.get(original_name, original_name)]: + for name in [original_name] + _flash_api_alternative_names.get(original_name, []): if supports_mapping[name]: kwargs_dict[name] = obj return name diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 1e77c8272cc3..14b15fce9fcf 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3788,7 +3788,7 @@ def from_pretrained( - `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)) - `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)) - `"flash_attention_4"` (using [Dao-AILab/flash-attention/flash_attn/cute](https://github.com/Dao-AILab/flash-attention/tree/main/flash_attn/cute)). - - `"flash_attention_torch"` (using a combination of `"sdpa"` and [`varlen`](https://docs.pytorch.org/docs/stable/nn.attention.varlen.html)). + - `"flash_attention_torch"` (using [`varlen`](https://docs.pytorch.org/docs/stable/nn.attention.varlen.html)). By default, if available, SDPA will be used. The default is otherwise the manual `"eager"` implementation. Accept HF kernel references in the form: From 7a0ce68dd9ad0c17bb3ebdd194fefb8e017a6f19 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 13:07:45 +0000 Subject: [PATCH 16/33] docstrings for support mapping --- src/transformers/modeling_flash_attention_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 02ea98670b16..951f50897b87 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -316,6 +316,11 @@ def _consolidate_flash_kwarg_alternative_name( The potential object to be passed as kwarg, depending on whether it is supported or not. original_name (`str`): The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. + supports_mapping (`dict[str, bool]`): + An dict containing the underlying information of the Flash Attention function on its supported parameters. + Keys are all valid parameters (and alternative names) across all standards and values are whether any of these + parameters could be found in their underlying signatures (in each case). For more information, please check + `_lazy_define_process_function` that creates this mapping. Return: name (`str`, *optional*): The associated name the object was added to the kwargs (if it was added; otherwise None). @@ -394,6 +399,11 @@ def _process_flash_attention_kwargs( The cumulative sequence lengths within the query tensor during a varlen forward. cu_seqlens_k (`torch.IntTensor]`, *optional*): The cumulative sequence lengths within the key/value tensor during a varlen forward. + supports_mapping (`dict[str, bool]`): + An dict containing the underlying information of the Flash Attention function on its supported parameters. + Keys are all valid parameters (and alternative names) across all standards and values are whether any of these + parameters could be found in their underlying signatures (in each case). For more information, please check + `_lazy_define_process_function` that creates this mapping. Return: flash_kwargs (`dict`): A dict of kwargs that are requested and supported. From f43f855a2cb523a1ea9183a92a2a181874bae038 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 13:16:13 +0000 Subject: [PATCH 17/33] fix these --- tests/utils/test_modeling_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index 7366845c4d78..3265c5fcc96c 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -2839,7 +2839,7 @@ def test_not_available_flash(self): _ = AutoModel.from_pretrained( "hf-internal-testing/tiny-random-GPTBigCodeModel", attn_implementation="flash_attention_2" ) - self.assertTrue("the package for FlashAttention2 doesn't seem to be installed." in str(cm.exception)) + self.assertTrue("the package for Flash Attention 2 doesn't seem to be installed." in str(cm.exception)) def test_not_available_flash_with_config(self): if is_flash_attn_2_available(): @@ -2862,7 +2862,7 @@ def test_not_available_flash_with_config(self): attn_implementation="flash_attention_2", ) - self.assertTrue("the package for FlashAttention2 doesn't seem to be installed." in str(cm.exception)) + self.assertTrue("the package for Flash Attention 2 doesn't seem to be installed." in str(cm.exception)) def test_kernels_fallback(self): if not is_kernels_available(): From 25bc50e081ffbf5f65772e2332583ea3dfa75ecb Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 18:49:32 +0000 Subject: [PATCH 18/33] important fix --- src/transformers/modeling_flash_attention_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 951f50897b87..0ff2b818e3b8 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -783,6 +783,8 @@ def _flash_attention_pure_varlen( position_ids: torch.Tensor, cu_seq_lens_q: torch.IntTensor | None, cu_seq_lens_k: torch.IntTensor | None, + max_length_q: int | None, + max_length_k: int | None, ): """ Goes natively through the FA varlen API as the input is already prepared into packed sequences. @@ -951,6 +953,8 @@ def _flash_attention_forward( position_ids=position_ids, cu_seq_lens_q=cu_seq_lens_q, cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, ) # No padding From 0e4f5ee73a3fadfc6dee6ec96d84e3f1ec477a21 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 18:50:38 +0000 Subject: [PATCH 19/33] revert qwen vl for now --- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 528260c5a3f5..3fac7b3fcbad 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -33,7 +33,6 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub -from ...integrations.flash_attention import flash_attention_forward from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer @@ -53,7 +52,6 @@ maybe_autocast, merge_with_config_defaults, ) -from ...utils.import_utils import is_flash_attn_torch_available from ...utils.output_capturing import capture_outputs from .configuration_qwen2_vl import Qwen2VLConfig, Qwen2VLTextConfig, Qwen2VLVisionConfig @@ -414,11 +412,11 @@ def forward( self.config._attn_implementation, eager_attention_forward ) - if is_flash_attention_requested(self.config, allow_torch=is_flash_attn_torch_available()): + if is_flash_attention_requested(self.config): # Flash Attention: Use cu_seqlens for variable length attention max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() # Force FA interface to call underlying SDPA's registered varlen - attn_output, _ = flash_attention_forward( + attn_output, _ = attention_interface( self, query_states, key_states, From aaff87269600d16deb9210fe8f0d22e8c56e38e9 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 18:52:05 +0000 Subject: [PATCH 20/33] forgot about this one --- src/transformers/models/qwen2_vl/modeling_qwen2_vl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py index 3fac7b3fcbad..6dc8755528d7 100644 --- a/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py +++ b/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py @@ -415,7 +415,6 @@ def forward( if is_flash_attention_requested(self.config): # Flash Attention: Use cu_seqlens for variable length attention max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() - # Force FA interface to call underlying SDPA's registered varlen attn_output, _ = attention_interface( self, query_states, From 91cde1c6de1b6d34197c86e94717db52443bc373 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 19:07:00 +0000 Subject: [PATCH 21/33] deprecate stuff properly --- .../modeling_flash_attention_utils.py | 14 ++++++-------- src/transformers/models/bark/modeling_bark.py | 17 ++--------------- .../models/diffllama/modeling_diffllama.py | 11 +---------- .../models/diffllama/modular_diffllama.py | 11 +---------- .../models/falcon/modeling_falcon.py | 14 +------------- .../models/gpt_neo/modeling_gpt_neo.py | 15 +-------------- src/transformers/models/gptj/modeling_gptj.py | 15 +-------------- .../modeling_kyutai_speech_to_text.py | 15 +-------------- src/transformers/models/mimi/modeling_mimi.py | 15 +-------------- src/transformers/models/moshi/modeling_moshi.py | 15 +-------------- .../models/nemotron/modeling_nemotron.py | 11 +---------- src/transformers/utils/__init__.py | 1 + src/transformers/utils/import_utils.py | 12 ++++++++++++ tests/generation/test_flash_attention_parity.py | 4 +++- tests/utils/test_modeling_utils.py | 5 ++--- 15 files changed, 35 insertions(+), 140 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 0ff2b818e3b8..1b6dfee1703c 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -33,6 +33,7 @@ is_torch_xpu_available, logging, ) +from .utils import is_flash_attn_available as utils_is_flash_attn_available from .utils.import_utils import PACKAGE_DISTRIBUTION_MAPPING, is_tracing @@ -48,16 +49,13 @@ def flash_attn_supports_top_left_mask(): return False -# TODO Deprecate when all models have the attention interface def is_flash_attn_available(): - return ( - is_flash_attn_torch_available() - or is_flash_attn_4_available() - or is_flash_attn_3_available() - or is_flash_attn_2_available() - or is_torch_npu_available() - or is_torch_xpu_available() + warnings.warn( + "`is_flash_attn_available` has been moved to `utils` (and `utils.import_utils`). " + "This alias will be removed in v5.8.", + FutureWarning, ) + return utils_is_flash_attn_available() class FlashAttentionKwargs(TypedDict, total=False): diff --git a/src/transformers/models/bark/modeling_bark.py b/src/transformers/models/bark/modeling_bark.py index e688affe2cdd..11be883580f1 100644 --- a/src/transformers/models/bark/modeling_bark.py +++ b/src/transformers/models/bark/modeling_bark.py @@ -28,8 +28,9 @@ BarkEosPrioritizerLogitsProcessor, SuppressTokensLogitsProcessor, ) +from ...integrations.flash_attention import get_target_dtype from ...masking_utils import create_bidirectional_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import CausalLMOutputWithPast, MaskedLMOutput from ...modeling_utils import PreTrainedModel @@ -55,11 +56,6 @@ ) -if is_flash_attn_available(): - from ...integrations.flash_attention import get_target_dtype - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -181,14 +177,6 @@ class BarkSelfFlashAttention2(BarkSelfAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def _split_heads(self, tensor, num_heads, attn_head_size): """ Splits hidden_size dim into attn_head_size and num_heads @@ -238,7 +226,6 @@ def forward( attention_mask, query_len, dropout=self.dropout if self.training else 0.0, - use_top_left_mask=self._flash_attn_uses_top_left_mask, is_causal=self.is_causal, target_dtype=target_dtype, ) diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index d80ccd572dc3..e5283b7c3b2a 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -33,7 +33,7 @@ from ...generation import GenerationMixin from ...integrations import use_kernel_forward_from_hub, use_kernel_func_from_hub from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import ( GenericForQuestionAnswering, GenericForSequenceClassification, @@ -285,14 +285,6 @@ class DiffLlamaFlashAttention2(DiffLlamaAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -375,7 +367,6 @@ def forward( position_ids=position_ids, dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), - use_top_left_mask=self._flash_attn_uses_top_left_mask, is_causal=self.is_causal, ) diff --git a/src/transformers/models/diffllama/modular_diffllama.py b/src/transformers/models/diffllama/modular_diffllama.py index 71087303c97e..a4d94e198070 100644 --- a/src/transformers/models/diffllama/modular_diffllama.py +++ b/src/transformers/models/diffllama/modular_diffllama.py @@ -21,7 +21,7 @@ from ... import initialization as init from ...cache_utils import Cache, StaticCache -from ...modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_utils import PreTrainedModel from ...utils import logging from ..gemma.modeling_gemma import GemmaForCausalLM @@ -160,14 +160,6 @@ class DiffLlamaFlashAttention2(DiffLlamaAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -250,7 +242,6 @@ def forward( position_ids=position_ids, dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), - use_top_left_mask=self._flash_attn_uses_top_left_mask, is_causal=self.is_causal, ) diff --git a/src/transformers/models/falcon/modeling_falcon.py b/src/transformers/models/falcon/modeling_falcon.py index 016b3209b6b1..528b387d89e8 100644 --- a/src/transformers/models/falcon/modeling_falcon.py +++ b/src/transformers/models/falcon/modeling_falcon.py @@ -27,7 +27,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, @@ -49,9 +49,6 @@ from .configuration_falcon import FalconConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - logger = logging.get_logger(__name__) @@ -434,14 +431,6 @@ class FalconFlashAttention2(FalconAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -516,7 +505,6 @@ def forward( position_ids=position_ids, dropout=attn_dropout, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_weights = attn_output.reshape(batch_size, query_length, self.num_heads * self.head_dim) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 5bf0bc584d85..83d686d342ff 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -22,7 +22,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -41,10 +41,6 @@ from .configuration_gpt_neo import GPTNeoConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -165,14 +161,6 @@ class GPTNeoFlashAttention2(GPTNeoSelfAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states, @@ -241,7 +229,6 @@ def forward( dropout=attn_dropout, softmax_scale=1.0, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_weights_reshaped = attn_output.reshape(bsz, query_length, self.num_heads * self.head_dim) diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index c92026f2f209..60ea05d1220b 100644 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -24,7 +24,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -37,10 +37,6 @@ from .configuration_gptj import GPTJConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -232,14 +228,6 @@ class GPTJFlashAttention2(GPTJAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.FloatTensor, @@ -342,7 +330,6 @@ def forward( query_length, dropout=attention_dropout, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) # Reshape outputs diff --git a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py index b16274332baf..d7caaff5b397 100644 --- a/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py +++ b/src/transformers/models/kyutai_speech_to_text/modeling_kyutai_speech_to_text.py @@ -31,7 +31,7 @@ from ...cache_utils import Cache, DynamicCache, StaticCache from ...generation import GenerationConfig, GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update @@ -43,10 +43,6 @@ from .configuration_kyutai_speech_to_text import KyutaiSpeechToTextConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -520,14 +516,6 @@ class KyutaiSpeechToTextFlashAttention2(KyutaiSpeechToTextAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -612,7 +600,6 @@ def forward( dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() diff --git a/src/transformers/models/mimi/modeling_mimi.py b/src/transformers/models/mimi/modeling_mimi.py index 30480d1d1c03..bc5d816eeef0 100644 --- a/src/transformers/models/mimi/modeling_mimi.py +++ b/src/transformers/models/mimi/modeling_mimi.py @@ -25,7 +25,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, StaticCache from ...masking_utils import create_sliding_window_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update @@ -35,10 +35,6 @@ from .configuration_mimi import MimiConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -745,14 +741,6 @@ class MimiFlashAttention2(MimiAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -835,7 +823,6 @@ def forward( dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() diff --git a/src/transformers/models/moshi/modeling_moshi.py b/src/transformers/models/moshi/modeling_moshi.py index a967445c18ec..9b2b43505e4b 100644 --- a/src/transformers/models/moshi/modeling_moshi.py +++ b/src/transformers/models/moshi/modeling_moshi.py @@ -27,7 +27,7 @@ from ...cache_utils import Cache, DynamicCache, StaticCache from ...generation import GenerationConfig, GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, ModelOutput, Seq2SeqLMOutput from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update @@ -38,10 +38,6 @@ from .configuration_moshi import MoshiConfig, MoshiDepthConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -518,14 +514,6 @@ class MoshiFlashAttention2(MoshiAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -610,7 +598,6 @@ def forward( dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() diff --git a/src/transformers/models/nemotron/modeling_nemotron.py b/src/transformers/models/nemotron/modeling_nemotron.py index b91b45ffd183..d80401cfcdd5 100644 --- a/src/transformers/models/nemotron/modeling_nemotron.py +++ b/src/transformers/models/nemotron/modeling_nemotron.py @@ -27,7 +27,7 @@ from ...cache_utils import Cache, DynamicCache, StaticCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import ( GenericForQuestionAnswering, GenericForSequenceClassification, @@ -306,14 +306,6 @@ class NemotronFlashAttention2(NemotronAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -394,7 +386,6 @@ def forward( position_ids=position_ids, dropout=dropout_rate, sliding_window=getattr(self, "sliding_window", None), - use_top_left_mask=self._flash_attn_uses_top_left_mask, is_causal=self.is_causal, ) diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 4431880b9320..9624c9dcf3ca 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -133,6 +133,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_available, is_flash_attn_greater_or_equal, is_flash_attn_greater_or_equal_2_10, is_flash_attn_torch_available, diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 6c02fce0d2b3..f5d301f5bef3 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -990,6 +990,18 @@ def is_flash_attn_torch_available() -> bool: return is_torch_greater_or_equal("2.11.0") and is_torch_cuda_available() +@lru_cache +def is_flash_attn_available(): + return ( + is_flash_attn_torch_available() + or is_flash_attn_4_available() + or is_flash_attn_3_available() + or is_flash_attn_2_available() + or is_torch_npu_available() + or is_torch_xpu_available() + ) + + @lru_cache def is_flash_attn_greater_or_equal(library_version: str) -> bool: is_available, flash_attn_version = _is_package_available("flash_attn", return_version=True) diff --git a/tests/generation/test_flash_attention_parity.py b/tests/generation/test_flash_attention_parity.py index 7b71ed02dc36..dcd9a546006b 100644 --- a/tests/generation/test_flash_attention_parity.py +++ b/tests/generation/test_flash_attention_parity.py @@ -80,7 +80,7 @@ def _benchmark_generation(self, model, inputs, n_warmup=3, n_runs=5): @require_all_flash_attn @pytest.mark.all_flash_attn_test def test_flash_attention_parity(self): - flash_attn_versions = [2, 3, 4] + flash_attn_versions = [2, 3, 4, "torch"] model_id = "meta-llama/Llama-3.2-1B-Instruct" prompt = ["The ETH AI Center is", "What is life?"] @@ -174,3 +174,5 @@ def generate(model, version, outputs, logits, logprobs): for idx, version in enumerate(flash_attn_versions): print(f" With FA{version}: {times[idx]}") print("---") + + assert False diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index 3265c5fcc96c..27777aa5898d 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -59,7 +59,6 @@ is_torch_available, logging, ) -from transformers.modeling_flash_attention_utils import is_flash_attn_available from transformers.models.mistral.modeling_mistral import MistralModel from transformers.testing_utils import ( TOKEN, @@ -755,7 +754,7 @@ def test_model_from_pretrained_attn_implementation(self): # 2. explicit from_pretrained's attn_implementation argument with a config argument attn_implementation_available = ["eager", "sdpa"] - if is_flash_attn_available(): + if is_flash_attn_2_available(): attn_implementation_available.append("flash_attention_2") if is_flash_attn_3_available(): @@ -783,7 +782,7 @@ def test_model_from_config_attn_implementation(self): # 3. config created with explicit attn_implementation and from_config overriding with explicit attn_implementation argument attn_implementation_available = ["eager", "sdpa"] - if is_flash_attn_available(): + if is_flash_attn_2_available(): attn_implementation_available.append("flash_attention_2") if is_flash_attn_3_available(): From 7ac60720e53ea41ba7273d9100c91645109e6eb8 Mon Sep 17 00:00:00 2001 From: Vasqu Date: Wed, 1 Apr 2026 19:45:51 +0000 Subject: [PATCH 22/33] fix --- src/transformers/modeling_flash_attention_utils.py | 4 +++- tests/generation/test_flash_attention_parity.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 1b6dfee1703c..80aa9df99998 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -415,7 +415,9 @@ def _process_flash_attention_kwargs( supports_mapping=supports_mapping, ) - deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + deterministic = ( + deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + ) for assignable_variable, original_variable_name in zip( [is_causal, dropout, softmax_scale, softcap, deterministic, s_aux, cu_seqlens_q, cu_seqlens_k], ["causal", "dropout_p", "softmax_scale", "softcap", "deterministic", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], diff --git a/tests/generation/test_flash_attention_parity.py b/tests/generation/test_flash_attention_parity.py index dcd9a546006b..7b71ed02dc36 100644 --- a/tests/generation/test_flash_attention_parity.py +++ b/tests/generation/test_flash_attention_parity.py @@ -80,7 +80,7 @@ def _benchmark_generation(self, model, inputs, n_warmup=3, n_runs=5): @require_all_flash_attn @pytest.mark.all_flash_attn_test def test_flash_attention_parity(self): - flash_attn_versions = [2, 3, 4, "torch"] + flash_attn_versions = [2, 3, 4] model_id = "meta-llama/Llama-3.2-1B-Instruct" prompt = ["The ETH AI Center is", "What is life?"] @@ -174,5 +174,3 @@ def generate(model, version, outputs, logits, logprobs): for idx, version in enumerate(flash_attn_versions): print(f" With FA{version}: {times[idx]}") print("---") - - assert False From 05aae16afcca2b4e6223233f7da86ba22aae6035 Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 19:32:03 +0200 Subject: [PATCH 23/33] fix deprecation cycle + add proper varlen path --- .../integrations/npu_flash_attention.py | 2 +- .../modeling_flash_attention_utils.py | 89 ++++++++++++------- 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/src/transformers/integrations/npu_flash_attention.py b/src/transformers/integrations/npu_flash_attention.py index 061b2fcecd25..62d8af23be54 100644 --- a/src/transformers/integrations/npu_flash_attention.py +++ b/src/transformers/integrations/npu_flash_attention.py @@ -46,7 +46,7 @@ def get_attn_mask_npu(device): def is_npu_fa2_top_left_aligned_causal_mask(): warnings.warn( - "`is_npu_fa2_top_left_aligned_causal_mask` is deprecated and will be removed in v5.8. " + "`is_npu_fa2_top_left_aligned_causal_mask` is deprecated and will be removed in v5.16. " "This is no longer needed as the minimum required FA version is no longer affected by this.", FutureWarning, ) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index ec74a0380ecc..cfaf3a0293c0 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -44,7 +44,7 @@ def flash_attn_supports_top_left_mask(): warnings.warn( - "`flash_attn_supports_top_left_mask` is deprecated and will be removed in v5.8. " + "`flash_attn_supports_top_left_mask` is deprecated and will be removed in v5.16. " "This is no longer needed as the minimum required FA version is no longer affected by this.", FutureWarning, ) @@ -54,7 +54,7 @@ def flash_attn_supports_top_left_mask(): def is_flash_attn_available(): warnings.warn( "`is_flash_attn_available` has been moved to `utils` (and `utils.import_utils`). " - "This alias will be removed in v5.8.", + "This alias will be removed in v5.16.", FutureWarning, ) return utils_is_flash_attn_available() @@ -673,10 +673,6 @@ def _flash_attention_mask_varlen( Manually unpads the tensors based on the attention mask, runs it through the varlen API and sticks back the to the original sequence lengths by padding with zeros where we manually removed padding values. """ - # Default create an all 1 (True) mask if nothing is given - if attention_mask is None: - attention_mask = torch.ones(size=(key_states.shape[:2]), device=key_states.device, dtype=torch.bool) - q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( query_states, key_states, value_states, attention_mask, query_length, unpad_fn ) @@ -856,6 +852,49 @@ def _flash_attention_pure( return out +def _flash_attention_pure_varlen_alt( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, +): + """Native FA varlen API without any padding or masks.""" + batch_size, max_length_q = query_states.shape[:2] + max_length_k = key_states.shape[1] + + cu_seq_lens_q = torch.arange( + batch_size + 1, + device=query_states.device, + dtype=torch.int32, + ) * max_length_q + + cu_seq_lens_k = torch.arange( + batch_size + 1, + device=key_states.device, + dtype=torch.int32, + ) * max_length_k + + q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) + k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) + v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) + + out = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), + ) + if isinstance(out, tuple): + out = out[0] + return out + + def fa_peft_integration_check( q: torch.Tensor, k: torch.Tensor, @@ -980,34 +1019,16 @@ def _flash_attention_forward( # No padding else: - # Case 1: No native base FA, we fallback to simulating varlen (but with no padding) - if flash_fn is None: - if not is_tracing(query_states): - logger.warning_once( - "We detected that your current underlying Flash Attention implementation does not implement a simple base" - "Flash Attention function (non-varlen). This can lead to slight inefficiencies (generation speed) and " - "changes in generation." - ) + # Varlen imitates no masks based on the seq len of the input + flash_fwd = flash_varlen_fn if flash_fn is None else flash_fn + flash_wrapper = _flash_attention_pure_varlen_alt if flash_fn is None else _flash_attention_pure - out = _flash_attention_mask_varlen( - flash_varlen_fn=flash_varlen_fn, - flash_kwargs=flash_kwargs, - unpad_fn=unpad_fn, - pad_fn=pad_fn, - query_states=query_states, - key_states=key_states, - value_states=value_states, - attention_mask=attention_mask, - query_length=query_length, - ) - # Case 2: Normal base FA as intended - else: - out = _flash_attention_pure( - flash_fn=flash_fn, - flash_kwargs=flash_kwargs, - query_states=query_states, - key_states=key_states, - value_states=value_states, - ) + out = flash_wrapper( + flash_fwd, + flash_kwargs=flash_kwargs, + query_states=query_states, + key_states=key_states, + value_states=value_states, + ) return out From 54e2bd0dc676e1d619398f6f5fc9b2590162da6a Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 19:36:02 +0200 Subject: [PATCH 24/33] style --- .../modeling_flash_attention_utils.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index cfaf3a0293c0..1061531fefc3 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -863,17 +863,23 @@ def _flash_attention_pure_varlen_alt( batch_size, max_length_q = query_states.shape[:2] max_length_k = key_states.shape[1] - cu_seq_lens_q = torch.arange( - batch_size + 1, - device=query_states.device, - dtype=torch.int32, - ) * max_length_q - - cu_seq_lens_k = torch.arange( - batch_size + 1, - device=key_states.device, - dtype=torch.int32, - ) * max_length_k + cu_seq_lens_q = ( + torch.arange( + batch_size + 1, + device=query_states.device, + dtype=torch.int32, + ) + * max_length_q + ) + + cu_seq_lens_k = ( + torch.arange( + batch_size + 1, + device=key_states.device, + dtype=torch.int32, + ) + * max_length_k + ) q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) From 151e7f364f1c6de3e71d0392a4d3c50c41ad8d76 Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 19:58:16 +0200 Subject: [PATCH 25/33] reshape at the end --- src/transformers/modeling_flash_attention_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 1061531fefc3..3f69644dbf22 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -898,7 +898,8 @@ def _flash_attention_pure_varlen_alt( ) if isinstance(out, tuple): out = out[0] - return out + + return out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) def fa_peft_integration_check( From 10e34c719a81e2dd108606fdfe880bd8c7c5f902 Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 20:15:54 +0200 Subject: [PATCH 26/33] tests --- .../self-scheduled-flash-attn-caller.yml | 2 +- conftest.py | 1 + pyproject.toml | 1 + src/transformers/testing_utils.py | 11 +++++++ .../generation/test_flash_attention_parity.py | 2 +- tests/generation/test_utils.py | 27 +++++++++++++++++ tests/models/gemma3/test_modeling_gemma3.py | 8 +++++ tests/models/sam3/test_modeling_sam3.py | 14 +++++++++ tests/test_modeling_common.py | 30 +++++++++++++++++++ tests/utils/test_modeling_utils.py | 8 +++++ 10 files changed, 102 insertions(+), 2 deletions(-) diff --git a/.github/workflows/self-scheduled-flash-attn-caller.yml b/.github/workflows/self-scheduled-flash-attn-caller.yml index 2d8f3c97a5d3..63155fe05cb2 100644 --- a/.github/workflows/self-scheduled-flash-attn-caller.yml +++ b/.github/workflows/self-scheduled-flash-attn-caller.yml @@ -62,5 +62,5 @@ jobs: runner_type: "a10" report_repo_id: hf-internal-testing/transformers_flash_attn_ci commit_sha: ${{ github.sha }} - pytest_marker: "flash_attn_test or flash_attn_3_test or flash_attn_4_test or all_flash_attn_test" + pytest_marker: "flash_attn_test or flash_attn_3_test or flash_attn_4_test or flash_attn_torch_test or all_flash_attn_test" secrets: inherit diff --git a/conftest.py b/conftest.py index 773f33d59b72..eab6df572152 100644 --- a/conftest.py +++ b/conftest.py @@ -251,6 +251,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "flash_attn_test: mark test which tests flash attention functionality") config.addinivalue_line("markers", "flash_attn_3_test: mark test which tests flash attention 3 functionality") config.addinivalue_line("markers", "flash_attn_4_test: mark test which tests flash attention 4 functionality") + config.addinivalue_line("markers", "flash_attn_torch_test: mark test which tests flash attention torch functionality") config.addinivalue_line( "markers", "all_flash_attn_test: mark test which tests all mainline flash attentions' functionality" ) diff --git a/pyproject.toml b/pyproject.toml index 1ff95cdd7904..96f50c59c916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ addopts = "--doctest-glob='**/*.md'" doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS" markers = [ "all_flash_attn_test: marks tests related to all mainline flash attentions at once (deselect with '-m \"not all_flash_attn_test\"')", + "flash_attn_torch_test: marks tests related to flash attention torch (deselect with '-m \"not flash_attn_torch_test\"')", "flash_attn_4_test: marks tests related to flash attention 4 (deselect with '-m \"not flash_attn_4_test\"')", "flash_attn_3_test: marks tests related to flash attention 3 (deselect with '-m \"not flash_attn_3_test\"')", "flash_attn_test: marks tests related to flash attention (deselect with '-m \"not flash_attn_test\"')", diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 0fd80aaac49a..5308fd98c7fc 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -97,6 +97,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_flash_linear_attention_available, is_flute_available, is_fouroversix_available, @@ -740,6 +741,15 @@ def require_flash_attn_4(test_case): return unittest.skipUnless(is_flash_attn_4_available(), "test requires Flash Attention 4")(test_case) +def require_flash_attn_torch(test_case): + """ + Decorator marking a test that requires Flash Attention Torch. + + These tests are skipped when Flash Attention Torch isn't installed. + """ + return unittest.skipUnless(is_flash_attn_torch_available(), "test requires Flash Attention Torch")(test_case) + + def require_all_flash_attn(test_case): flash_attn_available = is_flash_attn_2_available(kernels_fallback_ok=True) @@ -749,6 +759,7 @@ def require_all_flash_attn(test_case): flash_attn_available, is_flash_attn_3_available(), is_flash_attn_4_available(), + is_flash_attn_torch_available(), ) ), "test requires all mainline Flash Attention packages", diff --git a/tests/generation/test_flash_attention_parity.py b/tests/generation/test_flash_attention_parity.py index 7b71ed02dc36..b025cb8d2096 100644 --- a/tests/generation/test_flash_attention_parity.py +++ b/tests/generation/test_flash_attention_parity.py @@ -80,7 +80,7 @@ def _benchmark_generation(self, model, inputs, n_warmup=3, n_runs=5): @require_all_flash_attn @pytest.mark.all_flash_attn_test def test_flash_attention_parity(self): - flash_attn_versions = [2, 3, 4] + flash_attn_versions = [2, 3, 4, "torch"] model_id = "meta-llama/Llama-3.2-1B-Instruct" prompt = ["The ETH AI Center is", "What is life?"] diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index ddbd6863541c..48e39b07776f 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -46,6 +46,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_optimum_quanto, require_torch, require_torch_accelerator, @@ -1876,6 +1877,7 @@ def _test_attention_implementation(self, attn_implementation): "flash_attention_2": "_supports_flash_attn", "flash_attention_3": "_supports_flash_attn", "flash_attention_4": "_supports_flash_attn", + "flash_attention_torch": "_supports_flash_attn", } for model_class in self.all_generative_model_classes: @@ -1983,6 +1985,14 @@ def test_eager_matches_fa4_generate(self): """Tests that generate has equivalent outputs with FA4 and eager attention implementations.""" self._test_attention_implementation("flash_attention_4") + @pytest.mark.flash_attn_torch_test + @require_flash_attn_torch + @require_torch_gpu + @slow + def test_eager_matches_fa_torch_generate(self): + """Tests that generate has equivalent outputs with FA Torch and eager attention implementations.""" + self._test_attention_implementation("flash_attention_torch") + @require_flash_attn @require_torch_accelerator @pytest.mark.flash_attn_test @@ -2094,6 +2104,7 @@ def attention_mask_padding_matches_padding_free_with_position_ids( "flash_attention_2": "_supports_flash_attn", "flash_attention_3": "_supports_flash_attn", "flash_attention_4": "_supports_flash_attn", + "flash_attention_torch": "_supports_flash_attn", } for model_class in self.all_generative_model_classes: @@ -2257,6 +2268,22 @@ def test_flash_attention_4_padding_matches_padding_free_with_position_ids_and_fa attn_implementation="flash_attention_4", fa_kwargs=True ) + @require_flash_attn_torch + @require_torch_gpu + @pytest.mark.flash_attn_torch_test + @slow + def test_flash_attention_torch_padding_matches_padding_free_with_position_ids(self): + self.attention_mask_padding_matches_padding_free_with_position_ids(attn_implementation="flash_attention_torch") + + @require_flash_attn_torch + @require_torch_gpu + @pytest.mark.flash_attn_torch_test + @slow + def test_flash_attention_torch_padding_matches_padding_free_with_position_ids_and_fa_kwargs(self): + self.attention_mask_padding_matches_padding_free_with_position_ids( + attn_implementation="flash_attention_torch", fa_kwargs=True + ) + def _get_custom_4d_mask_test_data(self): # Sequence in which all but the last token is the same input_ids = torch.tensor( diff --git a/tests/models/gemma3/test_modeling_gemma3.py b/tests/models/gemma3/test_modeling_gemma3.py index 50ffd07f36f2..f729012cd591 100644 --- a/tests/models/gemma3/test_modeling_gemma3.py +++ b/tests/models/gemma3/test_modeling_gemma3.py @@ -37,6 +37,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_torch, require_torch_accelerator, require_torch_gpu, @@ -335,6 +336,13 @@ def test_flash_attn_3_from_config(self): def test_flash_attn_4_from_config(self): self.flash_attn_from_config(attn_implementation="flash_attention_4", test_fwd_in_train=False) + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + def test_flash_attn_torch_from_config(self): + self.flash_attn_from_config(attn_implementation="flash_attention_torch", test_fwd_in_train=False) + def test_attention_mask_composition(self): config = self.model_tester.get_config() config.text_config._attn_implementation = "eager" diff --git a/tests/models/sam3/test_modeling_sam3.py b/tests/models/sam3/test_modeling_sam3.py index 478f79ab7449..df9d62242a9e 100644 --- a/tests/models/sam3/test_modeling_sam3.py +++ b/tests/models/sam3/test_modeling_sam3.py @@ -640,6 +640,20 @@ def test_flash_attn_4_inference_equivalence(self): def test_flash_attn_4_inference_equivalence_right_padding(self): pass + @unittest.skip( + reason="Sam3Model creates attention masks from features (with gradients), " + "which is incompatible with flash attention's expectation of binary masks" + ) + def test_flash_attn_torch_inference_equivalence(self): + pass + + @unittest.skip( + reason="Sam3Model creates attention masks from features (with gradients), " + "which is incompatible with flash attention's expectation of binary masks" + ) + def test_flash_attn_torch_inference_equivalence_right_padding(self): + pass + @unittest.skip( reason="Sam3Model creates attention masks from features (with gradients), " "which is incompatible with flash attention's expectation of binary masks" diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 2d6f9b41133d..0729c10b7b57 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -97,6 +97,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_kernels, require_non_hpu, require_torch, @@ -3618,6 +3619,22 @@ def test_flash_attn_4_inference_equivalence(self): def test_flash_attn_4_inference_equivalence_right_padding(self): self.flash_attn_inference_equivalence(attn_implementation="flash_attention_4", padding_side="right") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + @is_flaky() + def test_flash_attn_torch_inference_equivalence(self): + self.flash_attn_inference_equivalence(attn_implementation="flash_attention_torch", padding_side="left") + + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + @is_flaky() + def test_flash_attn_torch_inference_equivalence_right_padding(self): + self.flash_attn_inference_equivalence(attn_implementation="flash_attention_torch", padding_side="right") + def test_attn_implementation_composite_models(self): """ Tests if composite models can receive a dict object as attn_implementation, where each key should be @@ -4040,6 +4057,12 @@ def test_flash_attn_3_can_dispatch_composite_models(self): def test_flash_attn_4_can_dispatch_composite_models(self): self.flash_attn_can_dispatch_composite_models(attn_implementation="flash_attention_4") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + def test_flash_attn_torch_can_dispatch_composite_models(self): + self.flash_attn_can_dispatch_composite_models(attn_implementation="flash_attention_torch") + @require_flash_attn @require_torch_accelerator @require_bitsandbytes @@ -4214,6 +4237,13 @@ def test_flash_attn_3_from_config(self): def test_flash_attn_4_from_config(self): self.flash_attn_from_config(attn_implementation="flash_attention_4") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + def test_flash_attn_torch_from_config(self): + self.flash_attn_from_config(attn_implementation="flash_attention_torch") + def test_sliding_window_mask(self): """Tests that we can control the sliding window attention behavior of a model.""" config, inputs = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index f9dd743cc6b5..422871883bff 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -89,6 +89,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_kernels_available, is_torch_npu_available, ) @@ -771,6 +772,9 @@ def test_model_from_pretrained_attn_implementation(self): if is_flash_attn_4_available(): attn_implementation_available.append("flash_attention_4") + if is_flash_attn_torch_available(): + attn_implementation_available.append("flash_attention_torch") + for requested_attn_implementation in attn_implementation_available: model = AutoModelForCausalLM.from_pretrained( TINY_MISTRAL, attn_implementation=requested_attn_implementation @@ -799,6 +803,9 @@ def test_model_from_config_attn_implementation(self): if is_flash_attn_4_available(): attn_implementation_available.append("flash_attention_4") + if is_flash_attn_torch_available(): + attn_implementation_available.append("flash_attention_torch") + for requested_attn_implementation in attn_implementation_available: config = AutoConfig.from_pretrained(TINY_MISTRAL, attn_implementation=requested_attn_implementation) # Ensure the config was set correctly @@ -3061,6 +3068,7 @@ def test_flash_attn_available_no_keyerror_when_missing_from_distribution_map(sel self.assertFalse(is_flash_attn_2_available()) self.assertFalse(is_flash_attn_3_available()) self.assertFalse(is_flash_attn_4_available()) + self.assertFalse(is_flash_attn_torch_available()) def test_not_available_flash_with_config(self): if is_flash_attn_2_available(): From 202e8735e26faf4c7aa3182f59a0bb116448117c Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 20:37:15 +0200 Subject: [PATCH 27/33] remove support for these models --- conftest.py | 4 +++- src/transformers/modeling_utils.py | 13 ++++++++++++- src/transformers/models/gemma2/modeling_gemma2.py | 1 + src/transformers/models/gemma2/modular_gemma2.py | 2 +- src/transformers/models/gemma3/modular_gemma3.py | 1 + src/transformers/models/gemma3n/modular_gemma3n.py | 1 + src/transformers/models/t5gemma/modeling_t5gemma.py | 1 + .../models/vaultgemma/modeling_vaultgemma.py | 1 + .../models/videoprism/modeling_videoprism.py | 1 + .../models/videoprism/modular_videoprism.py | 1 + 10 files changed, 23 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index eab6df572152..1fb74f043d0d 100644 --- a/conftest.py +++ b/conftest.py @@ -251,7 +251,9 @@ def pytest_configure(config): config.addinivalue_line("markers", "flash_attn_test: mark test which tests flash attention functionality") config.addinivalue_line("markers", "flash_attn_3_test: mark test which tests flash attention 3 functionality") config.addinivalue_line("markers", "flash_attn_4_test: mark test which tests flash attention 4 functionality") - config.addinivalue_line("markers", "flash_attn_torch_test: mark test which tests flash attention torch functionality") + config.addinivalue_line( + "markers", "flash_attn_torch_test: mark test which tests flash attention torch functionality" + ) config.addinivalue_line( "markers", "all_flash_attn_test: mark test which tests all mainline flash attentions' functionality" ) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 69a74125932e..e6e40abb681c 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1242,6 +1242,8 @@ class PreTrainedModel( _supports_flex_attn: bool = False # Model's compatible flash kernels (e.g., "kernels-community/flash-mla") defaulting to the first in the list _compatible_flash_implementations: list[str] | None = None + # Model's incompatible flash kernels (e.g., "flash_attention_torch") which will be rejected upon request + _incompatible_flash_implementations: list[str] | None = None # Advanced functionalities support supports_gradient_checkpointing: bool = False @@ -1861,7 +1863,6 @@ def _check_and_adjust_attn_implementation( attn_implementation = default_flash_implementation is_paged, base_implementation = split_attention_implementation(attn_implementation) - applicable_attn_implementation = attn_implementation requested_original_flash_attn = False @@ -1896,6 +1897,16 @@ def _check_and_adjust_attn_implementation( if is_paged: applicable_attn_implementation = f"paged|{applicable_attn_implementation}" + # E.g. fa torch cannot handle softcapping which is easier to set via this flag then setting all others to compatible + if ( + incompatible_flash_implementations := getattr(self, "_incompatible_flash_implementations", None) + ) is not None: + if base_implementation in incompatible_flash_implementations: + raise ValueError( + f"The requested flash attention implementation {applicable_attn_implementation} is not supported! " + f"See {incompatible_flash_implementations} for a list of incompatible implementations." + ) + if is_kernel(applicable_attn_implementation): try: # preload flash attention here to allow compile with fullgraph diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index 7e808722c60c..266292d1d541 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -377,6 +377,7 @@ class Gemma2PreTrainedModel(PreTrainedModel): "hidden_states": Gemma2DecoderLayer, "attentions": Gemma2Attention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/gemma2/modular_gemma2.py b/src/transformers/models/gemma2/modular_gemma2.py index 2edd9ef5f101..4d01767ba605 100644 --- a/src/transformers/models/gemma2/modular_gemma2.py +++ b/src/transformers/models/gemma2/modular_gemma2.py @@ -314,7 +314,7 @@ def forward( class Gemma2PreTrainedModel(GemmaPreTrainedModel): - pass + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping class Gemma2Model(GemmaModel): diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 7df6f5deb373..19a35c53e8dd 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -448,6 +448,7 @@ class Gemma3PreTrainedModel(Gemma2PreTrainedModel): "SiglipEncoderLayer", "SiglipMultiheadAttentionPoolingHead", ] + _incompatible_flash_implementations = AttributeError() # no softcap @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index d07c22acd35a..8c3e393ad53a 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -1653,6 +1653,7 @@ class Gemma3nPreTrainedModel(Gemma2PreTrainedModel): "hidden_states": Gemma3nTextDecoderLayer, "attentions": Gemma3nTextAttention, } + _incompatible_flash_implementations = AttributeError() # no softcap @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index 5609fa3b0b24..0a6f506c6f5e 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -567,6 +567,7 @@ class T5GemmaPreTrainedModel(PreTrainedModel): _supports_attention_backend = True # Recording is declared on T5GemmaEncoder/T5GemmaDecoder; None avoids inheriting the gemma2 dict _can_record_outputs = None + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/vaultgemma/modeling_vaultgemma.py b/src/transformers/models/vaultgemma/modeling_vaultgemma.py index a6def99e7a17..b94993a05977 100644 --- a/src/transformers/models/vaultgemma/modeling_vaultgemma.py +++ b/src/transformers/models/vaultgemma/modeling_vaultgemma.py @@ -368,6 +368,7 @@ class VaultGemmaPreTrainedModel(PreTrainedModel): "hidden_states": VaultGemmaDecoderLayer, "attentions": VaultGemmaAttention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/videoprism/modeling_videoprism.py b/src/transformers/models/videoprism/modeling_videoprism.py index 22772429e8b2..5cee67343345 100644 --- a/src/transformers/models/videoprism/modeling_videoprism.py +++ b/src/transformers/models/videoprism/modeling_videoprism.py @@ -486,6 +486,7 @@ class VideoPrismPreTrainedModel(PreTrainedModel): "hidden_states": VideoPrismLayer, "attentions": VideoPrismAttention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/videoprism/modular_videoprism.py b/src/transformers/models/videoprism/modular_videoprism.py index 4dadb55c3faa..a45176c27995 100644 --- a/src/transformers/models/videoprism/modular_videoprism.py +++ b/src/transformers/models/videoprism/modular_videoprism.py @@ -515,6 +515,7 @@ class VideoPrismPreTrainedModel(VivitPreTrainedModel): # sdpa is disabled because it does not support attention capping # used in eager and logits are too far off _supports_sdpa = False + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping _input_embed_layer = AttributeError() @torch.no_grad() From 3a5b41cf84367ea849d7099de0a59f67d73da42a Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 20:47:31 +0200 Subject: [PATCH 28/33] proper fixups on compatible fa implementations --- tests/generation/test_utils.py | 33 ++++++++++++++----- .../test_modeling_gemma4_unified.py | 3 ++ tests/test_modeling_common.py | 13 ++++++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index 48e39b07776f..573210764d03 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -1893,14 +1893,6 @@ def _test_attention_implementation(self, attn_implementation): inputs_dict[input_name] = input_data main_input = inputs_dict[model_class.main_input_name] - # FA doesn't accept masking in the middle of the sequence for now. We usually generate right-padded - # attention masks at test time and, with generate, the mask will be appended with 1s on the right, - # resulting in a mask with holes (not supported properly by FA). - if is_flash_attention_requested(requested_attention_implementation=attn_implementation): - for input_name in ("attention_mask", "decoder_attention_mask", "encoder_attention_mask"): - if input_name in inputs_dict: - inputs_dict[input_name] = torch.ones_like(inputs_dict[input_name]) - # make sure that all models have enough positions for generation if hasattr(config, "max_position_embeddings"): config.max_position_embeddings = max_new_tokens + main_input.shape[1] + 1 @@ -1908,6 +1900,22 @@ def _test_attention_implementation(self, attn_implementation): set_config_for_less_flaky_test(config) model = model_class(config) + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + + # FA doesn't accept masking in the middle of the sequence for now. We usually generate right-padded + # attention masks at test time and, with generate, the mask will be appended with 1s on the right, + # resulting in a mask with holes (not supported properly by FA). + for input_name in ("attention_mask", "decoder_attention_mask", "encoder_attention_mask"): + if input_name in inputs_dict: + inputs_dict[input_name] = torch.ones_like(inputs_dict[input_name]) + # If not all sub-models support flex, skip the test. We could potentially set not supported backbones # to "eager" attention, leaving it for future updates on multimodality tests sub_models_supporting_attn = [ @@ -2142,6 +2150,15 @@ def attention_mask_padding_matches_padding_free_with_position_ids( ): self.skipTest(f"At least some parts of {model_class.__name__} don't support {attn_implementation}") + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + if "position_ids" not in inspect.signature(model.forward).parameters: self.skipTest("Model does not support position_ids") diff --git a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py index 8314c2777303..40c8b8c0073e 100644 --- a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py +++ b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py @@ -469,6 +469,9 @@ def identity_forward(self, hidden_states): valid_fa_implementations = model._compatible_flash_implementations if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue # If we end up here, at least one model class was not skipped _has_run_at_least_one_model = True diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 0729c10b7b57..4b42532168e6 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -122,6 +122,7 @@ is_torch_bf16_available_on_device, is_torch_fp16_available_on_device, ) +from transformers.utils.generic import is_flash_attention_requested from transformers.utils.output_capturing import CompileableContextVar, OutputRecorder from .exporters.test_export import ExportTesterMixin @@ -3439,6 +3440,9 @@ def flash_attn_inference_equivalence( valid_fa_implementations = model._compatible_flash_implementations if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue # If we end up here, at least one model class was not skipped _has_run_at_least_one_model = True @@ -4181,6 +4185,15 @@ def flash_attn_from_config(self, attn_implementation: str, test_fwd_in_train: bo ): self.skipTest(reason=f"At least some parts of this model do not support {attn_implementation}") + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + # TODO: to change it in the future with other relevant auto classes fa_model = model_class._from_config( config, attn_implementation=attn_implementation, dtype=torch.bfloat16 From 5d06085ec0e2bb1d56a932fc437039f5573b00b6 Mon Sep 17 00:00:00 2001 From: vasqu Date: Tue, 28 Jul 2026 20:48:21 +0200 Subject: [PATCH 29/33] revert --- .github/workflows/self-scheduled-flash-attn-caller.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-scheduled-flash-attn-caller.yml b/.github/workflows/self-scheduled-flash-attn-caller.yml index 63155fe05cb2..2d8f3c97a5d3 100644 --- a/.github/workflows/self-scheduled-flash-attn-caller.yml +++ b/.github/workflows/self-scheduled-flash-attn-caller.yml @@ -62,5 +62,5 @@ jobs: runner_type: "a10" report_repo_id: hf-internal-testing/transformers_flash_attn_ci commit_sha: ${{ github.sha }} - pytest_marker: "flash_attn_test or flash_attn_3_test or flash_attn_4_test or flash_attn_torch_test or all_flash_attn_test" + pytest_marker: "flash_attn_test or flash_attn_3_test or flash_attn_4_test or all_flash_attn_test" secrets: inherit From 6aa8493473356ffafbc703a734fe4b44c24f6117 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 29 Jul 2026 16:31:20 +0200 Subject: [PATCH 30/33] remove sdpa/fa interleave option --- src/transformers/modeling_flash_attention_utils.py | 2 +- src/transformers/modeling_utils.py | 5 +---- src/transformers/utils/generic.py | 10 +++------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index 3f69644dbf22..f6c11fee87dd 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -198,7 +198,7 @@ def _lazy_imports( is_paged, implementation = split_attention_implementation(implementation) - if implementation in ["sdpa", "flash_attention_torch"]: + if implementation == "flash_attention_torch": from torch.nn.attention.varlen import varlen_attn as flash_attn_varlen_func flash_attn_func = None # not supported yet diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e6e40abb681c..31fae2d1193b 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1937,10 +1937,7 @@ def _check_and_adjust_attn_implementation( ) # preload flash attention here to allow compile with fullgraph - if is_flash_attention_requested( - requested_attention_implementation=applicable_attn_implementation, - allow_torch=is_flash_attn_torch_available(), - ): + if is_flash_attention_requested(requested_attention_implementation=applicable_attn_implementation): lazy_import_flash_attention(applicable_attn_implementation) return applicable_attn_implementation diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index e517e639ccb3..48c1ececfd16 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -267,10 +267,7 @@ def is_mlx_array(x) -> bool: def is_flash_attention_requested( - config=None, - requested_attention_implementation: str | None = None, - version: int | list[int] | None = None, - allow_torch: bool | None = None, + config=None, requested_attention_implementation: str | None = None, version: int | list[int] | None = None, ) -> bool: """ Checks whether some flavor of flash attention is requested or not. Optionally, checks for specific versions of @@ -282,7 +279,6 @@ def is_flash_attention_requested( The different versions of flash attention are usually - Implementations based on the original flash attention repo: https://github.com/Dao-AILab/flash-attention - Kernels implementations such as: https://huggingface.co/kernels-community/vllm-flash-attn3 - - Or native torch integrations (as of `torch 2.11.0`) if allowed via `allow_torch` """ if config is not None and requested_attention_implementation is not None: raise ValueError( @@ -305,8 +301,8 @@ def is_flash_attention_requested( version = [version] return any(re.match(r".*flash.*" + str(v), checked_attention_implementation) is not None for v in version) - # Otherwise, check whether "flash" is in the attention implementation or "sdpa" if `allow_torch` is enabled (flash attention via torch) - return "flash" in checked_attention_implementation or (allow_torch and "sdpa" in checked_attention_implementation) + # Otherwise, just check whether "flash" is in the attention implementation + return "flash" in checked_attention_implementation def get_max_seqlen( From c6df6713dcc2e410be78ec23bdca2cdd5511c5a0 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 29 Jul 2026 16:33:41 +0200 Subject: [PATCH 31/33] style --- src/transformers/modeling_utils.py | 1 - src/transformers/utils/generic.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 44785e9115f3..16d0dc09d19b 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -129,7 +129,6 @@ KERNELS_MAX_VERSION, KERNELS_MIN_VERSION, is_flash_attn_greater_or_equal, - is_flash_attn_torch_available, is_huggingface_hub_greater_or_equal, is_sagemaker_mp_enabled, is_torch_cuda_available, diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index 48c1ececfd16..6441a4d86455 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -267,7 +267,9 @@ def is_mlx_array(x) -> bool: def is_flash_attention_requested( - config=None, requested_attention_implementation: str | None = None, version: int | list[int] | None = None, + config=None, + requested_attention_implementation: str | None = None, + version: int | list[int] | None = None, ) -> bool: """ Checks whether some flavor of flash attention is requested or not. Optionally, checks for specific versions of From 8b77f4613441f8363a7d7c01c8f5c220e044cd79 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 29 Jul 2026 18:09:46 +0200 Subject: [PATCH 32/33] best effort fix --- tests/generation/test_utils.py | 95 ++++++++++++++++++ tests/models/blip_2/test_modeling_blip_2.py | 4 +- .../test_modeling_instructblip.py | 4 +- .../test_modeling_instructblipvideo.py | 4 +- tests/models/janus/test_modeling_janus.py | 18 ++-- tests/test_modeling_common.py | 97 ++----------------- 6 files changed, 118 insertions(+), 104 deletions(-) diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index 573210764d03..361552e48409 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -124,11 +124,103 @@ def is_moe_model(config): return getattr(config, "_experts_implementation", None) is not None +def _prepare_config_headdim(config, requested_dim): + """ + This method allows to update the head dim for all model types including + composite models and models that do not support head dim by themselves. + + Why? A lot of kernels including flex attention rely on triton for compilation. + However, triton cannot handle hidden dimensions of less than 16 for example. + (There are many more examples especially now that the `kernels` library is + supported) + """ + config = copy.deepcopy(config) + + def update_config_headdim(config, requested_dim): + # Flex Attention cannot use dropout + if hasattr(config, "attention_dropout"): + config.attention_dropout = 0.0 + if hasattr(config, "attention_probs_dropout_prob"): + config.attention_probs_dropout_prob = 0.0 + + # Update the head dim and try to update hidden size as well if present in config + # NOTE: some models may have none if the values in sub-config, thus we check for `Noneness` + head_dim = None + # On a heterogeneous config `head_dim` cannot be read globally, so probe for it without + # tripping the guard. Per-layer overrides are bumped below. + global_head_dim = config._getattr_without_heterogeneous_validation("head_dim", None) + if global_head_dim is not None: + head_dim = global_head_dim + config.head_dim = max(requested_dim, global_head_dim) + if config.is_heterogeneous and "head_dim" in config.per_layer_attributes: + overrides = copy.deepcopy(config._heterogeneity_spec.per_layer_overrides) + for layer_overrides in overrides.values(): + if "head_dim" in layer_overrides: + layer_overrides["head_dim"] = max(requested_dim, layer_overrides["head_dim"]) + config.per_layer_config = overrides + + cross_head_dim = None + if hasattr(config, "cross_head_dim") and config.cross_head_dim is not None: + cross_head_dim = config.cross_head_dim + config.cross_head_dim = max(requested_dim, config.cross_head_dim) + + if ( + getattr(config, "hidden_size", None) is not None + and getattr(config, "num_attention_heads", None) is not None + ): + # For some models, num_attention_heads is a list of ints: we take the max to maximize the multiplier + num_attn_heads = getattr(config, "num_attention_heads") + num_attn_heads = num_attn_heads if isinstance(num_attn_heads, int) else max(num_attn_heads) + head_dim = head_dim if head_dim is not None else config.hidden_size // num_attn_heads + config.hidden_size *= max(requested_dim // head_dim, 1) + + if ( + getattr(config, "decoder_hidden_size", None) is not None + and getattr(config, "decoder_num_attention_heads", None) is not None + ): + decoder_head_dim = config.decoder_hidden_size // config.decoder_num_attention_heads + config.decoder_hidden_size *= max(requested_dim // decoder_head_dim, 1) + + if ( + getattr(config, "cross_hidden_size", None) is not None + and getattr(config, "cross_num_attention_heads", None) is not None + ): + cross_head_dim = ( + cross_head_dim + if cross_head_dim is not None + else config.cross_hidden_size // config.cross_num_attention_heads + ) + config.cross_hidden_size *= max(requested_dim // cross_head_dim, 1) + + # 3d rope also depends on the head dim + # (we assume easy shapes here where we get to the requested head dim at least) + if ( + getattr(config, "rope_parameters", None) is not None + and len(config.rope_parameters.get("mrope_section", [])) > 0 + ): + scaling_factor = max(requested_dim // (sum(config.rope_parameters["mrope_section"]) * 2), 1) + config.rope_parameters["mrope_section"] = [ + section * scaling_factor for section in config.rope_parameters["mrope_section"] + ] + + # Update config values + update_config_headdim(config, requested_dim) + for key in config.sub_configs: + if getattr(config, key) is not None: + sub_config = getattr(config, key) + update_config_headdim(sub_config, requested_dim) + + return config + + class GenerationTesterMixin(ExportGenerateTesterMixin): input_name = "input_ids" model_tester = None max_new_tokens = 3 + # Workaround to make it static within each mixin (model, generate) -> one override for both + _prepare_config_headdim = staticmethod(_prepare_config_headdim) + def prepare_config_and_inputs_for_generate(self, batch_size=2): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -1916,6 +2008,9 @@ def _test_attention_implementation(self, attn_implementation): if input_name in inputs_dict: inputs_dict[input_name] = torch.ones_like(inputs_dict[input_name]) + # Some FA implementations need specific multiples at least + config = self._prepare_config_headdim(config, 16) + # If not all sub-models support flex, skip the test. We could potentially set not supported backbones # to "eager" attention, leaving it for future updates on multimodality tests sub_models_supporting_attn = [ diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 63f152220810..45c3a297b007 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -36,7 +36,7 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( TEST_EAGER_MATCHES_SDPA_INFERENCE_PARAMETERIZATION, @@ -69,7 +69,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 13d64642c1a3..956735c727a2 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -45,7 +45,7 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -67,7 +67,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 85035e6ece32..582f4bc30255 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -43,7 +43,7 @@ ) from transformers.utils import is_torch_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -65,7 +65,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/janus/test_modeling_janus.py b/tests/models/janus/test_modeling_janus.py index 13eabc162b1c..a7ad25f922a0 100644 --- a/tests/models/janus/test_modeling_janus.py +++ b/tests/models/janus/test_modeling_janus.py @@ -41,7 +41,7 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor from ...test_pipeline_mixin import PipelineTesterMixin @@ -55,6 +55,14 @@ from PIL import Image +def _prepare_janus_config_headdim(config, requested_dim): + config = _prepare_config_headdim(config, requested_dim) + # Sync projection_dim with text hidden_size since aligner output must match text embeddings + if hasattr(config, "vision_config") and hasattr(config, "text_config"): + config.vision_config.projection_dim = config.text_config.hidden_size + return config + + class JanusVisionText2TextModelTester: def __init__( self, @@ -206,13 +214,7 @@ def _prepare_config_headdim(config, requested_dim): Override to ensure vision_config.projection_dim stays in sync with text_config.hidden_size. The aligner projects vision features to text embedding dimension, so they must match. """ - from tests.test_modeling_common import ModelTesterMixin - - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) - # Sync projection_dim with text hidden_size since aligner output must match text embeddings - if hasattr(config, "vision_config") and hasattr(config, "text_config"): - config.vision_config.projection_dim = config.text_config.hidden_size - return config + return _prepare_janus_config_headdim(config, requested_dim) def setUp(self): self.model_tester = JanusVisionText2TextModelTester(self) diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 4b42532168e6..66fed7483ce6 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -126,7 +126,7 @@ from transformers.utils.output_capturing import CompileableContextVar, OutputRecorder from .exporters.test_export import ExportTesterMixin -from .generation.test_utils import GenerationTesterMixin +from .generation.test_utils import GenerationTesterMixin, _prepare_config_headdim if is_torch_available(): @@ -728,6 +728,9 @@ class ModelTesterMixin(ExportTesterMixin): _is_composite = False model_split_percents = [0.5, 0.7, 0.9] + # Workaround to make it static within each mixin (model, generate) -> one override for both + _prepare_config_headdim = staticmethod(_prepare_config_headdim) + # Note: for all mixins that utilize the Hub in some way, we should ensure that # they contain the `hub_retry` decorator in case of failures. def __init_subclass__(cls, **kwargs): @@ -4179,6 +4182,9 @@ def flash_attn_from_config(self, attn_implementation: str, test_fwd_in_train: bo self.skipTest(f"{model_class.__name__} does not support {attn_implementation}") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # Some FA implementations need specific multiples at least + config = self._prepare_config_headdim(config, 16) + model = model_class(config) # let's construct it here to see if any submodels can't support flash attn if not all( submodel._supports_flash_attn for submodel in model.modules() if isinstance(submodel, PreTrainedModel) @@ -4358,95 +4364,6 @@ def test_torch_compile_for_training(self): for name, param in model._orig_mod.named_parameters(): torch.testing.assert_close(param.grad.detach().cpu(), params[name], rtol=1e-4, atol=1e-4) - @staticmethod - def _prepare_config_headdim(config, requested_dim): - """ - This method allows to update the head dim for all model types including - composite models and models that do not support head dim by themselves. - - Why? A lot of kernels including flex attention rely on triton for compilation. - However, triton cannot handle hidden dimensions of less than 16 for example. - (There are many more examples especially now that the `kernels` library is - supported) - """ - config = copy.deepcopy(config) - - def update_config_headdim(config, requested_dim): - # Flex Attention cannot use dropout - if hasattr(config, "attention_dropout"): - config.attention_dropout = 0.0 - if hasattr(config, "attention_probs_dropout_prob"): - config.attention_probs_dropout_prob = 0.0 - - # Update the head dim and try to update hidden size as well if present in config - # NOTE: some models may have none if the values in sub-config, thus we check for `Noneness` - head_dim = None - # On a heterogeneous config `head_dim` cannot be read globally, so probe for it without - # tripping the guard. Per-layer overrides are bumped below. - global_head_dim = config._getattr_without_heterogeneous_validation("head_dim", None) - if global_head_dim is not None: - head_dim = global_head_dim - config.head_dim = max(requested_dim, global_head_dim) - if config.is_heterogeneous and "head_dim" in config.per_layer_attributes: - overrides = copy.deepcopy(config._heterogeneity_spec.per_layer_overrides) - for layer_overrides in overrides.values(): - if "head_dim" in layer_overrides: - layer_overrides["head_dim"] = max(requested_dim, layer_overrides["head_dim"]) - config.per_layer_config = overrides - - cross_head_dim = None - if hasattr(config, "cross_head_dim") and config.cross_head_dim is not None: - cross_head_dim = config.cross_head_dim - config.cross_head_dim = max(requested_dim, config.cross_head_dim) - - if ( - getattr(config, "hidden_size", None) is not None - and getattr(config, "num_attention_heads", None) is not None - ): - # For some models, num_attention_heads is a list of ints: we take the max to maximize the multiplier - num_attn_heads = getattr(config, "num_attention_heads") - num_attn_heads = num_attn_heads if isinstance(num_attn_heads, int) else max(num_attn_heads) - head_dim = head_dim if head_dim is not None else config.hidden_size // num_attn_heads - config.hidden_size *= max(requested_dim // head_dim, 1) - - if ( - getattr(config, "decoder_hidden_size", None) is not None - and getattr(config, "decoder_num_attention_heads", None) is not None - ): - decoder_head_dim = config.decoder_hidden_size // config.decoder_num_attention_heads - config.decoder_hidden_size *= max(requested_dim // decoder_head_dim, 1) - - if ( - getattr(config, "cross_hidden_size", None) is not None - and getattr(config, "cross_num_attention_heads", None) is not None - ): - cross_head_dim = ( - cross_head_dim - if cross_head_dim is not None - else config.cross_hidden_size // config.cross_num_attention_heads - ) - config.cross_hidden_size *= max(requested_dim // cross_head_dim, 1) - - # 3d rope also depends on the head dim - # (we assume easy shapes here where we get to the requested head dim at least) - if ( - getattr(config, "rope_parameters", None) is not None - and len(config.rope_parameters.get("mrope_section", [])) > 0 - ): - scaling_factor = max(requested_dim // (sum(config.rope_parameters["mrope_section"]) * 2), 1) - config.rope_parameters["mrope_section"] = [ - section * scaling_factor for section in config.rope_parameters["mrope_section"] - ] - - # Update config values - update_config_headdim(config, requested_dim) - for key in config.sub_configs: - if getattr(config, key) is not None: - sub_config = getattr(config, key) - update_config_headdim(sub_config, requested_dim) - - return config - @require_torch_accelerator def test_flex_attention_with_grads(self): for model_class in self.all_model_classes: From 8469405223568fb4bf7c9a5856608bd89c11756a Mon Sep 17 00:00:00 2001 From: vasqu Date: Thu, 30 Jul 2026 13:38:05 +0200 Subject: [PATCH 33/33] add amd kernel alternative --- src/transformers/modeling_flash_attention_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index f6c11fee87dd..782d15fcd106 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -168,7 +168,7 @@ class FlashAttentionKwargs(TypedDict, total=False): } # alternative names within the different flash attention APIs, e.g. for attention sinks _flash_api_alternative_names = { - "s_aux": ["learnable_sink"], + "s_aux": ["learnable_sink", "sink"], "cu_seqlens_q": ["cu_seq_q"], "cu_seqlens_k": ["cu_seq_k"], "max_seqlen_q": ["max_q"],