diff --git a/conftest.py b/conftest.py index 797178e963c4..d568dbcab3f4 100644 --- a/conftest.py +++ b/conftest.py @@ -291,6 +291,9 @@ def pytest_configure(config): config.addinivalue_line("markers", "flash_attn_test: mark test which tests flash attention functionality") config.addinivalue_line("markers", "flash_attn_3_test: mark test which tests flash attention 3 functionality") config.addinivalue_line("markers", "flash_attn_4_test: mark test which tests flash attention 4 functionality") + config.addinivalue_line( + "markers", "flash_attn_torch_test: mark test which tests flash attention torch functionality" + ) config.addinivalue_line( "markers", "all_flash_attn_test: mark test which tests all mainline flash attentions' functionality" ) diff --git a/pyproject.toml b/pyproject.toml index 56fdd1d20769..7a8c0d027778 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ addopts = "--doctest-glob='**/*.md'" doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS" markers = [ "all_flash_attn_test: marks tests related to all mainline flash attentions at once (deselect with '-m \"not all_flash_attn_test\"')", + "flash_attn_torch_test: marks tests related to flash attention torch (deselect with '-m \"not flash_attn_torch_test\"')", "flash_attn_4_test: marks tests related to flash attention 4 (deselect with '-m \"not flash_attn_4_test\"')", "flash_attn_3_test: marks tests related to flash attention 3 (deselect with '-m \"not flash_attn_3_test\"')", "flash_attn_test: marks tests related to flash attention (deselect with '-m \"not flash_attn_test\"')", diff --git a/src/transformers/integrations/flash_attention.py b/src/transformers/integrations/flash_attention.py index ac5e9642af67..1b031342d534 100644 --- a/src/transformers/integrations/flash_attention.py +++ b/src/transformers/integrations/flash_attention.py @@ -1,13 +1,11 @@ import torch -from ..modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask +from ..modeling_flash_attention_utils import _flash_attention_forward from ..utils import logging logger = logging.get_logger(__name__) -_use_top_left_mask = flash_attn_supports_top_left_mask() - def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtype: """If the query is in float32, return a target dtype compatible with flash attention. Return None otherwise.""" @@ -84,7 +82,6 @@ def flash_attention_forward( softmax_scale=scaling, sliding_window=sliding_window, softcap=softcap, - use_top_left_mask=_use_top_left_mask, target_dtype=target_dtype, attn_implementation=module.config._attn_implementation, layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None, diff --git a/src/transformers/integrations/npu_flash_attention.py b/src/transformers/integrations/npu_flash_attention.py index 2af3ba84e012..62d8af23be54 100644 --- a/src/transformers/integrations/npu_flash_attention.py +++ b/src/transformers/integrations/npu_flash_attention.py @@ -12,6 +12,7 @@ import math import os +import warnings import torch @@ -23,15 +24,14 @@ # FlashAttention2 is supported on Ascend NPU with down-right aligned causal mask by default. -# Set environment variable `NPU_FA2_SPARSE_MODE` to 2 when using top-left aligned causal mask. -TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2 +# It can set an environment variable `NPU_FA2_SPARSE_MODE` to control this behavior. +TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2 # Deprecated DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE = 3 SPARSE_MODE = int(os.getenv("NPU_FA2_SPARSE_MODE", default=DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE)) -if SPARSE_MODE not in [TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE, DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE]: +if SPARSE_MODE != DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE: raise ValueError( - "Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 2 (top-left aligned causal mask) " - "or 3 (down-right aligned causal mask)." + "Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 3 (down-right aligned causal mask)." ) ATTN_MASK_NPU_CACHE = {} @@ -45,7 +45,12 @@ def get_attn_mask_npu(device): def is_npu_fa2_top_left_aligned_causal_mask(): - return SPARSE_MODE == TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE if is_torch_npu_available() else False + warnings.warn( + "`is_npu_fa2_top_left_aligned_causal_mask` is deprecated and will be removed in v5.16. " + "This is no longer needed as the minimum required FA version is no longer affected by this.", + FutureWarning, + ) + return False def npu_flash_attn_func( diff --git a/src/transformers/masking_utils.py b/src/transformers/masking_utils.py index bff64e79935a..79e71eb92dcc 100644 --- a/src/transformers/masking_utils.py +++ b/src/transformers/masking_utils.py @@ -718,6 +718,7 @@ class AttentionMaskInterface(GeneralInterface): "flash_attention_2": flash_attention_mask, "flash_attention_3": flash_attention_mask, "flash_attention_4": flash_attention_mask, + "flash_attention_torch": flash_attention_mask, "flex_attention": flex_attention_mask, } diff --git a/src/transformers/modeling_flash_attention_utils.py b/src/transformers/modeling_flash_attention_utils.py index c7f7089ec169..034f9d42b13b 100644 --- a/src/transformers/modeling_flash_attention_utils.py +++ b/src/transformers/modeling_flash_attention_utils.py @@ -14,9 +14,10 @@ import importlib import inspect import os +import warnings from collections.abc import Callable from functools import partial -from typing import TypedDict +from typing import Any, TypedDict import torch import torch.nn.functional as F @@ -25,6 +26,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_rocm_platform, is_torch_cuda_available, is_torch_mlu_available, @@ -33,6 +35,7 @@ is_torch_xpu_available, logging, ) +from .utils import is_flash_attn_available as utils_is_flash_attn_available from .utils.generic import split_attention_implementation from .utils.import_utils import PACKAGE_DISTRIBUTION_MAPPING, is_tracing @@ -40,25 +43,43 @@ logger = logging.get_logger(__name__) -# TODO Deprecate when all models have the attention interface def flash_attn_supports_top_left_mask(): - if is_flash_attn_2_available() or is_flash_attn_3_available() or is_flash_attn_4_available(): - return False - - from .integrations.npu_flash_attention import is_npu_fa2_top_left_aligned_causal_mask - - return is_npu_fa2_top_left_aligned_causal_mask() + warnings.warn( + "`flash_attn_supports_top_left_mask` is deprecated and will be removed in v5.16. " + "This is no longer needed as the minimum required FA version is no longer affected by this.", + FutureWarning, + ) + return False -# TODO Deprecate when all models have the attention interface def is_flash_attn_available(): - return ( - is_flash_attn_4_available() - or is_flash_attn_3_available() - or is_flash_attn_2_available() - or is_torch_npu_available() - or is_torch_xpu_available() + warnings.warn( + "`is_flash_attn_available` has been moved to `utils` (and `utils.import_utils`). " + "This alias will be removed in v5.16.", + FutureWarning, ) + return utils_is_flash_attn_available() + + +class FlashAttentionKwargs(TypedDict, total=False): + """ + Keyword arguments for Flash Attention with Compile. + + Attributes: + cu_seq_lens_q (`torch.IntTensor`, *optional*) + Gets cumulative sequence length for query state. + cu_seq_lens_k (`torch.IntTensor`, *optional*) + Gets cumulative sequence length for key state. + max_length_q (`int`, *optional*): + Maximum sequence length for query state. + max_length_k (`int`, *optional*): + Maximum sequence length for key state. + """ + + cu_seq_lens_q: torch.IntTensor | None + cu_seq_lens_k: torch.IntTensor | None + max_length_q: int | None + max_length_k: int | None # Mapping from flash attention implementations to their kernel fallback repositories. @@ -120,6 +141,14 @@ def is_flash_attn_available(): "supported_devices": ((is_torch_cuda_available, "cuda"),), "cuda_min_major_version": 9, # Hopper }, + "torch": { + "flash_attn_version": "torch", + "general_availability_check": is_flash_attn_torch_available, + "pkg_availability_check": lambda *args, **kwargs: importlib.util.find_spec("torch") is not None, + "supported_devices": ( + (is_torch_cuda_available, "cuda"), + ), # Too uncertain on all features, restricting for now + }, } @@ -137,9 +166,17 @@ def is_flash_attn_available(): _hf_api_to_flash_mapping = { "dropout": "dropout_p", "sliding_window": "window_size", + "is_causal": "causal", } # alternative names within the different flash attention APIs, e.g. for attention sinks -_flash_api_alternative_names = {"s_aux": "learnable_sink"} +_flash_api_alternative_names = { + "s_aux": ["learnable_sink", "sink"], + "cu_seqlens_q": ["cu_seq_q"], + "cu_seqlens_k": ["cu_seq_k"], + "max_seqlen_q": ["max_q"], + "max_seqlen_k": ["max_k"], + "softmax_scale": ["scale"], +} def _lazy_imports( @@ -163,17 +200,25 @@ def _lazy_imports( is_paged, implementation = split_attention_implementation(implementation) - if (implementation == "flash_attention_2" and is_fa2) or ( + if implementation == "flash_attention_torch": + from torch.nn.attention.varlen import varlen_attn as flash_attn_varlen_func + + flash_attn_func = None # not supported yet + flash_attn_with_kvcache = None # not supported yet + + elif (implementation == "flash_attention_2" and is_fa2) or ( implementation is None and is_fa2 and not is_fa3 and not is_fa4 ): from flash_attn import flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache from flash_attn.bert_padding import pad_input, unpad_input + elif is_torch_npu_available(): # Package `flash-attn` is unavailable on Ascend NPU, which will cause ImportError # Flash-Attention2 related apis for Ascend NPU must be imported from `.integrations.npu_flash_attention` module from .integrations.npu_flash_attention import npu_flash_attn_func as flash_attn_func from .integrations.npu_flash_attention import npu_flash_attn_varlen_func as flash_attn_varlen_func from .integrations.npu_flash_attention import npu_flash_attn_with_kvcache as flash_attn_with_kvcache + else: if implementation == "flash_attention_3" or (implementation is None and is_fa3 and not is_fa4): from flash_attn_interface import flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache @@ -181,6 +226,7 @@ def _lazy_imports( from flash_attn.cute import flash_attn_func, flash_attn_varlen_func flash_attn_with_kvcache = None # not supported yet + # Kernels fallback else: from .integrations.hub_kernels import load_and_register_attn_kernel @@ -207,18 +253,6 @@ def _lazy_imports( f"Could not find the currently requested flash attention implementation at `{implementation}`." "Make sure that you request a valid kernel from the hub, e.g. `kernels-community/flash-attn2`." ) - if flash_attn_func is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` only supports varlen, i.e. " - "it can only be used with continuous batching and does not support the full functionality for " - "the base transformers generation methods." - ) - if flash_attn_with_kvcache is None: - logger.warning( - f"The loaded flash attention implementation at `{implementation}` does not support block tables, so" - " the full performances of continuous batching will not be achieved, only the varlen path will be " - "used." - ) return flash_attn_func, flash_attn_varlen_func, flash_attn_with_kvcache, pad_input, unpad_input @@ -241,8 +275,9 @@ def _lazy_define_process_function(flash_function): fa_param = _hf_api_to_flash_mapping.get(param, param) supports_mapping[fa_param] = fa_param in flash_parameters - if (fa_alternative_name := _flash_api_alternative_names.get(param, param)) != fa_param: - supports_mapping[fa_alternative_name] = fa_alternative_name in flash_parameters + if (fa_alternative_name := _flash_api_alternative_names.get(param, fa_param)) != fa_param: + for alternative_name in fa_alternative_name: + supports_mapping[alternative_name] = alternative_name in flash_parameters return partial(_process_flash_attention_kwargs, supports_mapping=supports_mapping) @@ -286,6 +321,174 @@ def lazy_import_paged_flash_attention(implementation: str | None, allow_all_kern return flash_attn_varlen_func, flash_attn_with_kvcache_fn +def _consolidate_flash_kwarg_alternative_name( + kwargs_dict: dict[str, Any], obj: Any, original_name: str, supports_mapping: dict[str, bool] +): + """ + Consolidates different naming conventions under all FA functions that we support by checking for all alternative names + and whether the FA function supports it. Based on that, it is added to the (existing) kwargs in-place or not. + + Args: + kwargs_dict (`dict[str, Any]`): + The current dict of collected kwargs to be passed to the underlying FA function. + obj (`Any`): + The potential object to be passed as kwarg, depending on whether it is supported or not. + original_name (`str`): + The kwarg name associated with the `obj`. This is based on first conventions from the original FA package. + supports_mapping (`dict[str, bool]`): + An dict containing the underlying information of the Flash Attention function on its supported parameters. + Keys are all valid parameters (and alternative names) across all standards and values are whether any of these + parameters could be found in their underlying signatures (in each case). For more information, please check + `_lazy_define_process_function` that creates this mapping. + Return: + name (`str`, *optional*): + The associated name the object was added to the kwargs (if it was added; otherwise None). + """ + if obj is None: + return None + + global _flash_api_alternative_names + for name in [original_name] + _flash_api_alternative_names.get(original_name, []): + if supports_mapping[name]: + kwargs_dict[name] = obj + return name + + # Torch in specific does not set it, ignore + if original_name != "causal": + # 2 variables with less impact, warn if possible + if original_name in ["dropout_p", "deterministic"] and not is_tracing(obj): + # Check if it goes against defaults + if (original_name == "dropout_p" and obj != 0.0) or (original_name == "deterministic" and obj is True): + logger.warning_once( + f"We detected the usage of {original_name} within Flash Attention. It is not supported in this version " + "of Flash Attention. This will be ignored but may lead to unintended behavior." + ) + # Core features that would highly influence the output + else: + raise ValueError( + f"Tried to assign {original_name}={obj} (or alternative names) to the underlying flash attention function. " + "But couldn't find it in its signature. Please make sure to disable this feature (e.g. dropout) or use a different " + "Flash Attention version (e.g. FA4 for attentions sinks in GPT Oss) that supports it." + ) + + return None + + +def _process_flash_attention_kwargs( + is_causal: bool, + dropout: float = 0.0, + softmax_scale: float | None = None, + sliding_window: int | None = None, + softcap: float | None = None, + deterministic: bool | None = None, + s_aux: torch.Tensor | None = None, + max_seqlen_q: int | torch.IntTensor | None = None, + max_seqlen_k: int | torch.IntTensor | None = None, + cu_seqlens_q: torch.IntTensor | None = None, + cu_seqlens_k: torch.IntTensor | None = None, + supports_mapping: dict[str, bool] | None = None, + **kwargs, +): + """ + Returns a set of kwargs that are passed down to the according flash attention function based on + requested features and whether it is supported - depends on the version and kernel implementation + which is dynamically configured at `lazy_import_flash_attention`. The (un)supported features can be + inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. + + Args: + is_causal (`bool`): + Whether we perform causal (decoder) attention or full attention. + dropout (`float`): + Attention dropout. + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. + sliding_window (`int`, *optional*): + The size of the sliding window, i.e. we look at a max of `sliding_window` tokens back. + softcap (`float`, *optional*): + Softcap for the attention logits, used e.g. in gemma2. + deterministic (`bool`, *optional*): + Determines if the deterministic option introduced in flash_attn>=2.4.1 is enabled. + s_aux (`torch.Tensor`, *optional*): + Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head. + max_seqlen_q (`Union[int, torch.IntTensor]`, *optional*): + The maximum sequence length in the query tensor during a varlen forward. + max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): + The maximum sequence length in the key/value tensor during a varlen forward. + cu_seqlens_q (`torch.IntTensor]`, *optional*): + The cumulative sequence lengths within the query tensor during a varlen forward. + cu_seqlens_k (`torch.IntTensor]`, *optional*): + The cumulative sequence lengths within the key/value tensor during a varlen forward. + supports_mapping (`dict[str, bool]`): + An dict containing the underlying information of the Flash Attention function on its supported parameters. + Keys are all valid parameters (and alternative names) across all standards and values are whether any of these + parameters could be found in their underlying signatures (in each case). For more information, please check + `_lazy_define_process_function` that creates this mapping. + Return: + flash_kwargs (`dict`): + A dict of kwargs that are requested and supported. + """ + flash_kwargs = {} + + # Every API determined they needed their own convention... + consolidate_flash_kwarg_alternative_name = partial( + _consolidate_flash_kwarg_alternative_name, + kwargs_dict=flash_kwargs, + supports_mapping=supports_mapping, + ) + + deterministic = ( + deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" + ) + for assignable_variable, original_variable_name in zip( + [is_causal, dropout, softmax_scale, softcap, deterministic, s_aux, cu_seqlens_q, cu_seqlens_k], + ["causal", "dropout_p", "softmax_scale", "softcap", "deterministic", "s_aux", "cu_seqlens_q", "cu_seqlens_k"], + ): + consolidate_flash_kwarg_alternative_name( + obj=assignable_variable, + original_name=original_variable_name, + ) + + if supports_mapping["window_size"] and sliding_window is not None: + # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left + # and the current token for a total size of 5. However, we usually define our window sizes by + # their total window size (when causal). Encoder models as of now seldom use SWA and when they + # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. + flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) + + # Torch varlen can use sliding window but also has to set it to determine causality + if flash_kwargs.get("causal") is None: + if flash_kwargs.get("window_size") is None: + flash_kwargs["window_size"] = (-1, 0) if is_causal else (-1, -1) + elif is_causal: + flash_kwargs["window_size"] = (flash_kwargs["window_size"][0], 0) + + # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` + # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. + # + # You can either set + # - Env: `TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1` + # - Before compiling: `torch._dynamo.config.capture_scalar_outputs = True` + # to allow torch compile to handle scalar outputs in those cases. + same_max_seqlen = max_seqlen_q is max_seqlen_k # to avoid 2x device syncs + + assigned_name_q = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_q, original_name="max_seqlen_q") + if assigned_name_q is not None: + if not isinstance(max_seqlen_q, int) and is_tracing(max_seqlen_q): + max_seqlen_q = max_seqlen_q.item() + flash_kwargs[assigned_name_q] = max_seqlen_q + + assigned_name_k = consolidate_flash_kwarg_alternative_name(obj=max_seqlen_k, original_name="max_seqlen_k") + if assigned_name_k is not None: + if same_max_seqlen: + flash_kwargs[assigned_name_k] = flash_kwargs[assigned_name_q] + else: + if not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): + max_seqlen_k = max_seqlen_k.item() + flash_kwargs[assigned_name_k] = max_seqlen_k + + return flash_kwargs + + def _index_first_axis(tensor, indices): """ A local implementation of the PyTorch indexing operation `tensor[indices]` on the first axis, @@ -457,6 +660,63 @@ def _upad_input( ) +def _flash_attention_mask_varlen( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + unpad_fn: Callable, + pad_fn: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: torch.Tensor | None, + query_length: int, +): + """ + Manually unpads the tensors based on the attention mask, runs it through the varlen API and sticks + back the to the original sequence lengths by padding with zeros where we manually removed padding values. + """ + q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( + query_states, key_states, value_states, attention_mask, query_length, unpad_fn + ) + + # TODO for now this is required to work with + # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py + if "mps" in str(q.device): + cu_seq_lens_k = cu_seq_lens_k.clone() + + out_unpad = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), + ) + if isinstance(out_unpad, tuple): + out_unpad = out_unpad[0] + + return pad_fn(out_unpad, indices_q, query_states.size(0), query_length) + + +def _is_packed_sequence(position_ids, batch_size): + """ + Check the position ids whether packed sequences are indicated or not + 1. Position ids exist + 2. Flattened sequences only are supported + 3. Compile-friendly `not (torch.diff(position_ids, dim=-1) >= 0).all()`, i.e. we have multiple increasing sequences + """ + if position_ids is None: + return False + + increasing_position_sequences = ( + torch.arange(position_ids.shape[1], device=position_ids.device) + position_ids.min() + ) + return batch_size == 1 and (increasing_position_sequences - position_ids).abs().sum().bool() + + def prepare_fa_kwargs_from_position_ids(position_ids): """ This function returns all the necessary kwargs to call `flash_attn_varlen_func` extracted from position_ids. @@ -535,20 +795,115 @@ def _prepare_from_posids(query, key, value, position_ids): return (query, key, value, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k)) -def _is_packed_sequence(position_ids, batch_size): +def _flash_attention_pure_varlen( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + position_ids: torch.Tensor, + cu_seq_lens_q: torch.IntTensor | None, + cu_seq_lens_k: torch.IntTensor | None, + max_length_q: int | None, + max_length_k: int | None, +): """ - Check the position ids whether packed sequences are indicated or not - 1. Position ids exist - 2. Flattened sequences only are supported - 3. Compile-friendly `not (torch.diff(position_ids, dim=-1) >= 0).all()`, i.e. we have multiple increasing sequences + Goes natively through the FA varlen API as the input is already prepared into packed sequences. + The necessary information is already given (`cu_seq_lens`) or we infer it (`position_ids`) ourselves. """ - if position_ids is None: - return False + if cu_seq_lens_q is None or cu_seq_lens_k is None: + q, k, v, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _prepare_from_posids( + query_states, key_states, value_states, position_ids + ) + else: + q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) + k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) + v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) + + # TODO for now this is required to work with + # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py + if "mps" in str(q.device): + cu_seq_lens_k = cu_seq_lens_k.clone() + + out = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), + ) + if isinstance(out, tuple): + out = out[0] - increasing_position_sequences = ( - torch.arange(position_ids.shape[1], device=position_ids.device) + position_ids.min() + return out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) + + +def _flash_attention_pure( + flash_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, +): + """Native FA API without any padding or masks.""" + out = flash_fn(query_states, key_states, value_states, **flash_kwargs()) + if isinstance(out, tuple): + out = out[0] + return out + + +def _flash_attention_pure_varlen_alt( + flash_varlen_fn: Callable, + flash_kwargs: Callable, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, +): + """Native FA varlen API without any padding or masks.""" + batch_size, max_length_q = query_states.shape[:2] + max_length_k = key_states.shape[1] + + cu_seq_lens_q = ( + torch.arange( + batch_size + 1, + device=query_states.device, + dtype=torch.int32, + ) + * max_length_q ) - return batch_size == 1 and (increasing_position_sequences - position_ids).abs().sum().bool() + + cu_seq_lens_k = ( + torch.arange( + batch_size + 1, + device=key_states.device, + dtype=torch.int32, + ) + * max_length_k + ) + + q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) + k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) + v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) + + out = flash_varlen_fn( + q, + k, + v, + **flash_kwargs( + cu_seqlens_q=cu_seq_lens_q, + cu_seqlens_k=cu_seq_lens_k, + max_seqlen_q=max_length_q, + max_seqlen_k=max_length_k, + ), + ) + if isinstance(out, tuple): + out = out[0] + + return out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) def fa_peft_integration_check( @@ -569,130 +924,6 @@ def fa_peft_integration_check( return q, k, v -class FlashAttentionKwargs(TypedDict, total=False): - """ - Keyword arguments for Flash Attention with Compile. - - Attributes: - cu_seq_lens_q (`torch.LongTensor`, *optional*) - Gets cumulative sequence length for query state. - cu_seq_lens_k (`torch.LongTensor`, *optional*) - Gets cumulative sequence length for key state. - max_length_q (`int`, *optional*): - Maximum sequence length for query state. - max_length_k (`int`, *optional*): - Maximum sequence length for key state. - """ - - cu_seq_lens_q: torch.LongTensor | None - cu_seq_lens_k: torch.LongTensor | None - max_length_q: int | None - max_length_k: int | None - - -def _process_flash_attention_kwargs( - query_length: int, - key_length: int, - is_causal: bool, - dropout: float = 0.0, - softmax_scale: float | None = None, - sliding_window: int | None = None, - use_top_left_mask: bool = False, - softcap: float | None = None, - deterministic: bool | None = None, - s_aux: torch.Tensor | None = None, - max_seqlen_q: int | torch.IntTensor | None = None, - max_seqlen_k: int | torch.IntTensor | None = None, - supports_mapping: dict[str, bool] | None = None, - **kwargs, -): - """ - Returns a set of kwargs that are passed down to the according flash attention function based on - requested features and whether it is supported - depends on the version and kernel implementation - which is dynamically configured at `lazy_import_flash_attention`. The (un)supported features can be - inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. - - Args: - query_length (`int`): - Length of the query states - key_length (`int`): - Length of the key states - is_causal (`bool`): - Whether we perform causal (decoder) attention or full attention. - dropout (`float`): - Attention dropout. - softmax_scale (`float`, *optional*): - The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. - sliding_window (`int`, *optional*): - The size of the sliding window, i.e. we look at a max of `sliding_window` tokens back. - use_top_left_mask (`bool`): - Deprecated behavior of older versions of flash attention requiring different masking. - softcap (`float`, *optional*): - Softcap for the attention logits, used e.g. in gemma2. - deterministic (`bool`, *optional*): - Determines if the deterministic option introduced in flash_attn>=2.4.1 is enabled. - s_aux (`torch.Tensor`, *optional*): - Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head. - max_seqlen_q (`Union[int, torch.IntTensor]`, *optional*): - The maximum sequence length in the query tensor during a varlen forward. - max_seqlen_k (`Union[int, torch.IntTensor]`, *optional*): - The maximum sequence length in the key/value tensor during a varlen forward. - Return: - flash_kwargs (`dict`): - A dict of kwargs that are requested and supported. - """ - flash_kwargs = { - "causal": is_causal and not (use_top_left_mask and query_length == 1), - "softmax_scale": softmax_scale, - } - - if supports_mapping["dropout_p"]: - flash_kwargs["dropout_p"] = dropout - - if supports_mapping["window_size"] and sliding_window is not None and key_length > sliding_window: - # The flash attention API sets inclusive boundaries, i.e. (4, 0) would take 4 tokens to the left - # and the current token for a total size of 5. However, we usually define our window sizes by - # their total window size (when causal). Encoder models as of now seldom use SWA and when they - # do, they must align with this symmetric logic, i.e. for a total of `2*sliding_window + 1`. - flash_kwargs["window_size"] = (sliding_window - 1, sliding_window - 1) - - if supports_mapping["deterministic"]: - flash_kwargs["deterministic"] = ( - deterministic if deterministic is not None else os.getenv("FLASH_ATTENTION_DETERMINISTIC", "0") == "1" - ) - - if supports_mapping["softcap"] and softcap is not None: - flash_kwargs["softcap"] = softcap - - if ((legacy_sink_param := supports_mapping["s_aux"]) or supports_mapping["learnable_sink"]) and s_aux is not None: - if legacy_sink_param: - flash_kwargs["s_aux"] = s_aux # e.g. FA3 (vllm) - else: - flash_kwargs["learnable_sink"] = s_aux # FA4 - - # There is a limitation of the flash attention API, as the function `flash_attn_varlen_func` - # may require `max_length_q`, `max_length_k` to be passed as `int` and not `torch.Tensor`. - # - # You can either set - # - Env: `TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1` - # - Before compiling: `torch._dynamo.config.capture_scalar_outputs = True` - # to allow torch compile to handle scalar outputs in those cases. - same_max_seqlen = max_seqlen_q is max_seqlen_k # to avoid 2x device syncs - if supports_mapping["max_seqlen_q"] and max_seqlen_q is not None: - if not isinstance(max_seqlen_q, int) and is_tracing(max_seqlen_q): - max_seqlen_q = max_seqlen_q.item() - flash_kwargs["max_seqlen_q"] = max_seqlen_q - - if supports_mapping["max_seqlen_k"] and max_seqlen_k is not None: - if same_max_seqlen and flash_kwargs["max_seqlen_q"] is not None: - max_seqlen_k = flash_kwargs["max_seqlen_q"] - elif not isinstance(max_seqlen_k, int) and is_tracing(max_seqlen_k): - max_seqlen_k = max_seqlen_k.item() - flash_kwargs["max_seqlen_k"] = max_seqlen_k - - return flash_kwargs - - def _flash_attention_forward( query_states: torch.Tensor, key_states: torch.Tensor, @@ -704,11 +935,10 @@ def _flash_attention_forward( position_ids: torch.Tensor | None = None, softmax_scale: float | None = None, sliding_window: int | None = None, - use_top_left_mask: bool = False, softcap: float | None = None, deterministic: bool | None = None, - cu_seq_lens_q: torch.LongTensor | None = None, - cu_seq_lens_k: torch.LongTensor | None = None, + cu_seq_lens_q: torch.IntTensor | None = None, + cu_seq_lens_k: torch.IntTensor | None = None, max_length_q: int | None = None, max_length_k: int | None = None, target_dtype: torch.dtype | None = None, @@ -746,18 +976,17 @@ def _flash_attention_forward( # Extract the flash attention kwargs that have been requested (and are supported by the implementation) flash_kwargs = partial( process_flash_kwargs_fn, - query_length=query_length, - key_length=key_states.size(1), is_causal=is_causal, dropout=dropout, softmax_scale=softmax_scale, sliding_window=sliding_window, - use_top_left_mask=use_top_left_mask, softcap=softcap, deterministic=deterministic, **kwargs, ) + # Determine which FA function to call + # We will use `flash_varlen_fn` to prevent cross-example attention and also allow padding free approach under two cases: # Case 1. If position ids is provided and the position ids indicate packed sequences, see `_is_packed_sequence`. # Case 2. Some models pass directly pre-computed `cu_seqlens` so we don't need to infer it from position ids. It is safe to @@ -772,61 +1001,45 @@ def _flash_attention_forward( # Contains at least one padding token in the sequence if attention_mask is not None: - q, k, v, indices_q, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _upad_input( - query_states, key_states, value_states, attention_mask, query_length, unpad_fn - ) - - # TODO for now this is required to work with - # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py - if "mps" in str(q.device): - cu_seq_lens_k = cu_seq_lens_k.clone() - - out_unpad = flash_varlen_fn( - q, - k, - v, - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - **flash_kwargs(max_seqlen_q=max_length_q, max_seqlen_k=max_length_k), + out = _flash_attention_mask_varlen( + flash_varlen_fn=flash_varlen_fn, + flash_kwargs=flash_kwargs, + unpad_fn=unpad_fn, + pad_fn=pad_fn, + query_states=query_states, + key_states=key_states, + value_states=value_states, + attention_mask=attention_mask, + query_length=query_length, ) - if isinstance(out_unpad, tuple): - out_unpad = out_unpad[0] - - out = pad_fn(out_unpad, indices_q, query_states.size(0), query_length) # Padding free, i.e. sequences flattened into one total sequence elif is_fa_with_varlen_kwargs or is_fa_with_position_ids: - if cu_seq_lens_q is None or cu_seq_lens_k is None: - q, k, v, (cu_seq_lens_q, cu_seq_lens_k), (max_length_q, max_length_k) = _prepare_from_posids( - query_states, key_states, value_states, position_ids - ) - else: - q = query_states.reshape(-1, query_states.size(-2), query_states.size(-1)) - k = key_states.reshape(-1, key_states.size(-2), key_states.size(-1)) - v = value_states.reshape(-1, value_states.size(-2), value_states.size(-1)) - - # TODO for now this is required to work with - # https://huggingface.co/kernels-community/metal-flash-sdpa/blob/main/torch-ext/metal_flash_sdpa/__init__.py - if "mps" in str(q.device): - cu_seq_lens_k = cu_seq_lens_k.clone() - - out = flash_varlen_fn( - q, - k, - v, - cu_seqlens_q=cu_seq_lens_q, - cu_seqlens_k=cu_seq_lens_k, - **flash_kwargs(max_seqlen_q=max_length_q, max_seqlen_k=max_length_k), + out = _flash_attention_pure_varlen( + flash_varlen_fn=flash_varlen_fn, + flash_kwargs=flash_kwargs, + query_states=query_states, + key_states=key_states, + value_states=value_states, + position_ids=position_ids, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, ) - if isinstance(out, tuple): - out = out[0] - - out = out.view(query_states.size(0), -1, out.size(-2), out.size(-1)) # No padding else: - out = flash_fn(query_states, key_states, value_states, **flash_kwargs()) - if isinstance(out, tuple): - out = out[0] + # Varlen imitates no masks based on the seq len of the input + flash_fwd = flash_varlen_fn if flash_fn is None else flash_fn + flash_wrapper = _flash_attention_pure_varlen_alt if flash_fn is None else _flash_attention_pure + + out = flash_wrapper( + flash_fwd, + flash_kwargs=flash_kwargs, + query_states=query_states, + key_states=key_states, + value_states=value_states, + ) return out diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index b11807f8915a..6481561fd1f7 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -133,6 +133,7 @@ is_huggingface_hub_greater_or_equal, is_sagemaker_mp_enabled, is_torch_cuda_available, + is_torch_greater_or_equal, is_tracing, ) from .utils.loading_report import LoadStateDictInfo, log_state_dict_report @@ -1130,6 +1131,8 @@ class PreTrainedModel( _supports_flex_attn: bool = False # Model's compatible flash kernels (e.g., "kernels-community/flash-mla") defaulting to the first in the list _compatible_flash_implementations: list[str] | None = None + # Model's incompatible flash kernels (e.g., "flash_attention_torch") which will be rejected upon request + _incompatible_flash_implementations: list[str] | None = None # Set to `False` by models that can never run under context parallelism, whatever their config # (attention sinks, for instance, which SDPA cannot express). Models whose *config* rules it out are @@ -1513,7 +1516,7 @@ def can_generate(cls) -> bool: def _flash_attn_import_error( self, - flash_attn_version: int, + flash_attn_version: int | str, general_availability_check: Callable, pkg_availability_check: Callable, supported_devices: tuple[tuple[Callable, str], ...], @@ -1525,7 +1528,7 @@ def _flash_attn_import_error( on why it failed - package import and/or device incompatibility issues. Args: - flash_attn_version (`int`): + flash_attn_version (`Union[int, str]`): The requested version of Flash Attention. general_availability_check (`Callable`): Checks whether our `is_available` function detects the specific FA version. Failing reasons @@ -1552,29 +1555,33 @@ def _flash_attn_import_error( return if not general_availability_check(): - preface = f"FlashAttention{flash_attn_version} has been toggled on, but it cannot be used due to the following error:" + preface = f"Flash Attention {flash_attn_version} has been toggled on, but it cannot be used due to the following error:" # Can the package be seen in the import structure if not pkg_availability_check(): raise ImportError( - f"{preface} the package for FlashAttention{flash_attn_version} doesn't seem to be installed." + f"{preface} the package for Flash Attention {flash_attn_version} doesn't seem to be installed." ) - # Minimum version (FA2 only) + # Minimum version elif flash_attn_version == 2 and not is_flash_attn_greater_or_equal("2.3.3"): - raise ImportError(f"{preface} FlashAttention{flash_attn_version} requires at least version `2.3.3`.") + raise ImportError(f"{preface} Flash Attention {flash_attn_version} requires at least version `2.3.3`.") + elif flash_attn_version == "torch" and not is_torch_greater_or_equal("2.11.0"): + raise ImportError( + f"{preface} Flash Attention {flash_attn_version} requires at least version `2.11.0`." + ) else: # Supported devices availability device_availability_checks, device_names = zip(*supported_devices) if not any(device_availability_check() for device_availability_check in device_availability_checks): raise ImportError( - f"{preface} FlashAttention{flash_attn_version} is not available on CPU. Please make sure you are on any of the supported devices: {device_names}." + f"{preface} Flash Attention {flash_attn_version} is not available on CPU. Please make sure you are on any of the supported devices: {device_names}." ) # Cuda major versions (more recent FA versions are specialized to newer cuda devices) elif cuda_min_major_version is not None and is_torch_cuda_available(): major, _ = torch.cuda.get_device_capability() if major < cuda_min_major_version: raise ImportError( - f"{preface} FlashAttention{flash_attn_version} requires compute capability >= {cuda_min_major_version}, but found {torch.cuda.get_device_capability()} with compute capability {major}.x" + f"{preface} Flash Attention {flash_attn_version} requires compute capability >= {cuda_min_major_version}, but found {torch.cuda.get_device_capability()} with compute capability {major}.x" ) def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool = False) -> bool: @@ -1597,7 +1604,7 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new" ) - if flash_attn_version not in [2, 3, 4]: + if flash_attn_version not in [2, 3, 4, "torch"]: raise ValueError(f"Requested Flash Attention {flash_attn_version} which is not supported.") # Check if we can even use the FA version based on the env of the user @@ -1605,7 +1612,7 @@ def _flash_attn_can_dispatch(self, flash_attn_version: int, is_init_check: bool # Check for attention dropout, which is incompatible with newer FA versions # (many should not really care about dropout as it is not super effective, hence warning for now) - if flash_attn_version > 2: + if flash_attn_version == "torch" or flash_attn_version > 2: if hasattr(self.config, "attention_dropout") and self.config.attention_dropout > 0: logger.warning_once( f"You are attempting to use Flash Attention {flash_attn_version} with dropout. " @@ -1759,13 +1766,16 @@ def _check_and_adjust_attn_implementation( attn_implementation = default_flash_implementation is_paged, base_implementation = split_attention_implementation(attn_implementation) - applicable_attn_implementation = attn_implementation requested_original_flash_attn = False if is_flash_attention_requested(requested_attention_implementation=base_implementation): # If FA not installed, do not fail but use kernels instead if possible for fa_version in FLASH_ATTENTION_COMPATIBILITY_MATRIX.keys(): + # No fallback for native torch + if fa_version == "torch": + continue + # Check whether we have an original FA requested but not available in the env if ( base_implementation == f"flash_attention_{fa_version}" @@ -1790,6 +1800,16 @@ def _check_and_adjust_attn_implementation( if is_paged: applicable_attn_implementation = f"paged|{applicable_attn_implementation}" + # E.g. fa torch cannot handle softcapping which is easier to set via this flag then setting all others to compatible + if ( + incompatible_flash_implementations := getattr(self, "_incompatible_flash_implementations", None) + ) is not None: + if base_implementation in incompatible_flash_implementations: + raise ValueError( + f"The requested flash attention implementation {applicable_attn_implementation} is not supported! " + f"See {incompatible_flash_implementations} for a list of incompatible implementations." + ) + if is_kernel(applicable_attn_implementation): try: # preload flash attention here to allow compile with fullgraph @@ -1859,9 +1879,11 @@ def get_correct_attn_implementation(self, requested_attention: str | None, is_in # Perform relevant checks if is_flash_attention_requested(requested_attention_implementation=applicable_attention) and ( - fa_matched := re.search(r"^flash_attention_(\d)$", applicable_attention) + fa_matched := re.search(r"^flash_attention_(torch|\d)$", applicable_attention) ): - fa_version = int(fa_matched.group(1)) # last digit + fa_version = ( + fa_matched.group(1) if fa_matched.group(1) == "torch" else int(fa_matched.group(1)) + ) # last digit self._flash_attn_can_dispatch(flash_attn_version=fa_version, is_init_check=is_init_check) elif "flex_attention" in applicable_attention: self._flex_attn_can_dispatch(is_init_check) @@ -3912,6 +3934,7 @@ def from_pretrained( - `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)) - `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)) - `"flash_attention_4"` (using [Dao-AILab/flash-attention/flash_attn/cute](https://github.com/Dao-AILab/flash-attention/tree/main/flash_attn/cute)). + - `"flash_attention_torch"` (using [`varlen`](https://docs.pytorch.org/docs/stable/nn.attention.varlen.html)). By default, if available, SDPA will be used. The default is otherwise the manual `"eager"` implementation. Accept HF kernel references in the form: @@ -5107,11 +5130,13 @@ class AttentionInterface(GeneralInterface): # Class instance object, so that a call to `register` can be reflected into all other files correctly, even if # a new instance is created (in order to locally override a given function) _global_mapping = { + "flash_attention_torch": flash_attention_forward, "flash_attention_4": flash_attention_forward, "flash_attention_3": flash_attention_forward, "flash_attention_2": flash_attention_forward, "flex_attention": flex_attention_forward, "sdpa": sdpa_attention_forward, + "paged|flash_attention_torch": flash_attention_forward, "paged|flash_attention_4": paged_attention_forward, "paged|flash_attention_3": paged_attention_forward, "paged|flash_attention_2": paged_attention_forward, @@ -5119,7 +5144,7 @@ class AttentionInterface(GeneralInterface): "paged|eager": eager_paged_attention_forward, } - def get_interface(self, attn_implementation: str, default: Callable) -> Callable: + def get_interface(self, attn_implementation: str | None, default: Callable) -> Callable: """Return the requested `attn_implementation`. Also strictly check its validity, and raise if invalid.""" if attn_implementation is None: logger.warning_once( @@ -5131,6 +5156,7 @@ def get_interface(self, attn_implementation: str, default: Callable) -> Callable raise KeyError( f"`{attn_implementation}` is not a valid attention implementation registered in the `AttentionInterface`" ) + return super().get(attn_implementation, default) diff --git a/src/transformers/models/bark/modeling_bark.py b/src/transformers/models/bark/modeling_bark.py index b775bcc793cf..34e1e0db683d 100644 --- a/src/transformers/models/bark/modeling_bark.py +++ b/src/transformers/models/bark/modeling_bark.py @@ -28,8 +28,9 @@ BarkEosPrioritizerLogitsProcessor, SuppressTokensLogitsProcessor, ) +from ...integrations.flash_attention import get_target_dtype from ...masking_utils import create_bidirectional_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import CausalLMOutputWithPast, MaskedLMOutput from ...modeling_utils import PreTrainedModel @@ -54,11 +55,6 @@ ) -if is_flash_attn_available(): - from ...integrations.flash_attention import get_target_dtype - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -180,14 +176,6 @@ class BarkSelfFlashAttention2(BarkSelfAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def _split_heads(self, tensor, num_heads, attn_head_size): """ Splits hidden_size dim into attn_head_size and num_heads @@ -237,7 +225,6 @@ def forward( attention_mask, query_len, dropout=self.dropout if self.training else 0.0, - use_top_left_mask=self._flash_attn_uses_top_left_mask, is_causal=self.is_causal, target_dtype=target_dtype, ) diff --git a/src/transformers/models/falcon/modeling_falcon.py b/src/transformers/models/falcon/modeling_falcon.py index e248b367c1ea..2a6c537273ba 100644 --- a/src/transformers/models/falcon/modeling_falcon.py +++ b/src/transformers/models/falcon/modeling_falcon.py @@ -26,7 +26,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, @@ -49,9 +49,6 @@ from .configuration_falcon import FalconConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - logger = logging.get_logger(__name__) @@ -429,14 +426,6 @@ class FalconFlashAttention2(FalconAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.Tensor, @@ -511,7 +500,6 @@ def forward( position_ids=position_ids, dropout=attn_dropout, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_weights = attn_output.reshape(batch_size, query_length, self.num_heads * self.head_dim) diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index 3c93b5ac2320..84bf824b802d 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -366,6 +366,7 @@ class Gemma2PreTrainedModel(PreTrainedModel): "hidden_states": Gemma2DecoderLayer, "attentions": Gemma2Attention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/gemma2/modular_gemma2.py b/src/transformers/models/gemma2/modular_gemma2.py index 814e80de817a..1db744b109b5 100644 --- a/src/transformers/models/gemma2/modular_gemma2.py +++ b/src/transformers/models/gemma2/modular_gemma2.py @@ -316,7 +316,7 @@ def forward( class Gemma2PreTrainedModel(GemmaPreTrainedModel): - pass + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping class Gemma2Model(GemmaModel): diff --git a/src/transformers/models/gemma3/modular_gemma3.py b/src/transformers/models/gemma3/modular_gemma3.py index 1078ff623f66..f73bd9ca3672 100644 --- a/src/transformers/models/gemma3/modular_gemma3.py +++ b/src/transformers/models/gemma3/modular_gemma3.py @@ -444,6 +444,7 @@ class Gemma3PreTrainedModel(Gemma2PreTrainedModel): base_model_prefix = "model" input_modalities = ("image", "text") _no_split_modules = ["Gemma3DecoderLayer"] + _incompatible_flash_implementations = AttributeError() # no softcap @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 53b81f3b491e..972d323083e7 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -1645,6 +1645,7 @@ class Gemma3nPreTrainedModel(Gemma2PreTrainedModel): "hidden_states": Gemma3nTextDecoderLayer, "attentions": Gemma3nTextAttention, } + _incompatible_flash_implementations = AttributeError() # no softcap @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index dfdbe5d57cbc..083517c4083d 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -22,7 +22,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -41,10 +41,6 @@ from .configuration_gpt_neo import GPTNeoConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -165,14 +161,6 @@ class GPTNeoFlashAttention2(GPTNeoSelfAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states, @@ -241,7 +229,6 @@ def forward( dropout=attn_dropout, softmax_scale=1.0, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) attn_weights_reshaped = attn_output.reshape(bsz, query_length, self.num_heads * self.head_dim) diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py index 277c08d4b4e2..bcf71f20de73 100644 --- a/src/transformers/models/gptj/modeling_gptj.py +++ b/src/transformers/models/gptj/modeling_gptj.py @@ -24,7 +24,7 @@ from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin from ...masking_utils import create_causal_mask -from ...modeling_flash_attention_utils import flash_attn_supports_top_left_mask, is_flash_attn_available +from ...modeling_flash_attention_utils import _flash_attention_forward from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import ( BaseModelOutputWithPast, @@ -37,10 +37,6 @@ from .configuration_gptj import GPTJConfig -if is_flash_attn_available(): - from ...modeling_flash_attention_utils import _flash_attention_forward - - logger = logging.get_logger(__name__) @@ -232,14 +228,6 @@ class GPTJFlashAttention2(GPTJAttention): flash attention and deal with padding tokens in case the input contains any of them. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. - # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. - # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). - self._flash_attn_uses_top_left_mask = flash_attn_supports_top_left_mask() - def forward( self, hidden_states: torch.FloatTensor, @@ -342,7 +330,6 @@ def forward( query_length, dropout=attention_dropout, is_causal=self.is_causal, - use_top_left_mask=self._flash_attn_uses_top_left_mask, ) # Reshape outputs diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index d6e5bb781a7c..1e3c7f1165de 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -556,6 +556,7 @@ class T5GemmaPreTrainedModel(PreTrainedModel): _supports_attention_backend = True # Recording is declared on T5GemmaEncoder/T5GemmaDecoder; None avoids inheriting the gemma2 dict _can_record_outputs = None + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/vaultgemma/modeling_vaultgemma.py b/src/transformers/models/vaultgemma/modeling_vaultgemma.py index 10fa163ea8ba..f565bb5e04b2 100644 --- a/src/transformers/models/vaultgemma/modeling_vaultgemma.py +++ b/src/transformers/models/vaultgemma/modeling_vaultgemma.py @@ -357,6 +357,7 @@ class VaultGemmaPreTrainedModel(PreTrainedModel): "hidden_states": VaultGemmaDecoderLayer, "attentions": VaultGemmaAttention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/videoprism/modeling_videoprism.py b/src/transformers/models/videoprism/modeling_videoprism.py index 185e4e83e0fc..4d5142591e3d 100644 --- a/src/transformers/models/videoprism/modeling_videoprism.py +++ b/src/transformers/models/videoprism/modeling_videoprism.py @@ -486,6 +486,7 @@ class VideoPrismPreTrainedModel(PreTrainedModel): "hidden_states": VideoPrismLayer, "attentions": VideoPrismAttention, } + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping @torch.no_grad() def _init_weights(self, module): diff --git a/src/transformers/models/videoprism/modular_videoprism.py b/src/transformers/models/videoprism/modular_videoprism.py index 3d00d9799fc4..4d52e797b307 100644 --- a/src/transformers/models/videoprism/modular_videoprism.py +++ b/src/transformers/models/videoprism/modular_videoprism.py @@ -515,6 +515,7 @@ class VideoPrismPreTrainedModel(VivitPreTrainedModel): # sdpa is disabled because it does not support attention capping # used in eager and logits are too far off _supports_sdpa = False + _incompatible_flash_implementations = ["flash_attention_torch"] # softcapping _input_embed_layer = AttributeError() @torch.no_grad() diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 4f2910f6c8e2..361bb8c15c1d 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -98,6 +98,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_flash_linear_attention_available, is_flute_available, is_fouroversix_available, @@ -746,6 +747,15 @@ def require_flash_attn_4(test_case): return unittest.skipUnless(is_flash_attn_4_available(), "test requires Flash Attention 4")(test_case) +def require_flash_attn_torch(test_case): + """ + Decorator marking a test that requires Flash Attention Torch. + + These tests are skipped when Flash Attention Torch isn't installed. + """ + return unittest.skipUnless(is_flash_attn_torch_available(), "test requires Flash Attention Torch")(test_case) + + def require_all_flash_attn(test_case): flash_attn_available = is_flash_attn_2_available(kernels_fallback_ok=True) @@ -755,6 +765,7 @@ def require_all_flash_attn(test_case): flash_attn_available, is_flash_attn_3_available(), is_flash_attn_4_available(), + is_flash_attn_torch_available(), ) ), "test requires all mainline Flash Attention packages", diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 12d2ece089c9..a5b2e566d261 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -140,8 +140,10 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_available, is_flash_attn_greater_or_equal, is_flash_attn_greater_or_equal_2_10, + is_flash_attn_torch_available, is_flash_linear_attention_available, is_flute_available, is_fouroversix_available, diff --git a/src/transformers/utils/generic.py b/src/transformers/utils/generic.py index dd24e82b7157..a1c090f7089e 100644 --- a/src/transformers/utils/generic.py +++ b/src/transformers/utils/generic.py @@ -267,7 +267,9 @@ def is_mlx_array(x) -> bool: def is_flash_attention_requested( - config=None, requested_attention_implementation: str | None = None, version: int | list[int] | None = None + config=None, + requested_attention_implementation: str | None = None, + version: int | list[int] | None = None, ) -> bool: """ Checks whether some flavor of flash attention is requested or not. Optionally, checks for specific versions of @@ -301,7 +303,7 @@ def is_flash_attention_requested( version = [version] return any(re.match(r".*flash.*" + str(v), checked_attention_implementation) is not None for v in version) - # Otherwise, just check "flash" is in the attention implementation + # Otherwise, just check whether "flash" is in the attention implementation return "flash" in checked_attention_implementation @@ -838,9 +840,9 @@ class TransformersKwargs(TypedDict, total=False): Turn this on to return the intermediary attention scores. output_router_logits (`Optional[bool]`, *optional*): For MoE models, this allows returning the router logits to compute the loss. - cu_seq_lens_q (`torch.LongTensor`, *optional*) + cu_seq_lens_q (`torch.IntTensor`, *optional*) Gets cumulative sequence length for query state. - cu_seq_lens_k (`torch.LongTensor`, *optional*) + cu_seq_lens_k (`torch.IntTensor`, *optional*) Gets cumulative sequence length for key state. max_length_q (`int`, *optional*): Maximum sequence length for query state. @@ -858,8 +860,8 @@ class TransformersKwargs(TypedDict, total=False): output_hidden_states: bool | None output_attentions: bool | None output_router_logits: bool | None - cu_seq_lens_q: torch.LongTensor | None - cu_seq_lens_k: torch.LongTensor | None + cu_seq_lens_q: torch.IntTensor | None + cu_seq_lens_k: torch.IntTensor | None max_length_q: int | None max_length_k: int | None position_ids: torch.LongTensor | None diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 03ab57887021..b27297fbb65b 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -1264,6 +1264,23 @@ def is_flash_attn_4_available() -> bool: return is_available and is_torch_cuda_available() +@lru_cache +def is_flash_attn_torch_available() -> bool: + return is_torch_greater_or_equal("2.11.0") and is_torch_cuda_available() + + +@lru_cache +def is_flash_attn_available(): + return ( + is_flash_attn_torch_available() + or is_flash_attn_4_available() + or is_flash_attn_3_available() + or is_flash_attn_2_available() + or is_torch_npu_available() + or is_torch_xpu_available() + ) + + @lru_cache def is_flash_attn_greater_or_equal(library_version: str) -> bool: is_available, flash_attn_version = _is_package_available("flash_attn", return_version=True) diff --git a/tests/generation/test_flash_attention_parity.py b/tests/generation/test_flash_attention_parity.py index 7b71ed02dc36..b025cb8d2096 100644 --- a/tests/generation/test_flash_attention_parity.py +++ b/tests/generation/test_flash_attention_parity.py @@ -80,7 +80,7 @@ def _benchmark_generation(self, model, inputs, n_warmup=3, n_runs=5): @require_all_flash_attn @pytest.mark.all_flash_attn_test def test_flash_attention_parity(self): - flash_attn_versions = [2, 3, 4] + flash_attn_versions = [2, 3, 4, "torch"] model_id = "meta-llama/Llama-3.2-1B-Instruct" prompt = ["The ETH AI Center is", "What is life?"] diff --git a/tests/generation/test_utils.py b/tests/generation/test_utils.py index ac1c78a0aa92..79a376e25acf 100644 --- a/tests/generation/test_utils.py +++ b/tests/generation/test_utils.py @@ -46,6 +46,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_optimum_quanto, require_torch, require_torch_accelerator, @@ -128,11 +129,103 @@ def is_moe_model(config): return getattr(config, "_experts_implementation", None) is not None +def _prepare_config_headdim(config, requested_dim): + """ + This method allows to update the head dim for all model types including + composite models and models that do not support head dim by themselves. + + Why? A lot of kernels including flex attention rely on triton for compilation. + However, triton cannot handle hidden dimensions of less than 16 for example. + (There are many more examples especially now that the `kernels` library is + supported) + """ + config = copy.deepcopy(config) + + def update_config_headdim(config, requested_dim): + # Flex Attention cannot use dropout + if hasattr(config, "attention_dropout"): + config.attention_dropout = 0.0 + if hasattr(config, "attention_probs_dropout_prob"): + config.attention_probs_dropout_prob = 0.0 + + # Update the head dim and try to update hidden size as well if present in config + # NOTE: some models may have none if the values in sub-config, thus we check for `Noneness` + head_dim = None + # On a heterogeneous config `head_dim` cannot be read globally, so probe for it without + # tripping the guard. Per-layer overrides are bumped below. + global_head_dim = config._getattr_without_heterogeneous_validation("head_dim", None) + if global_head_dim is not None: + head_dim = global_head_dim + config.head_dim = max(requested_dim, global_head_dim) + if config.is_heterogeneous and "head_dim" in config.per_layer_attributes: + overrides = copy.deepcopy(config._heterogeneity_spec.per_layer_overrides) + for layer_overrides in overrides.values(): + if "head_dim" in layer_overrides: + layer_overrides["head_dim"] = max(requested_dim, layer_overrides["head_dim"]) + config.per_layer_config = overrides + + cross_head_dim = None + if hasattr(config, "cross_head_dim") and config.cross_head_dim is not None: + cross_head_dim = config.cross_head_dim + config.cross_head_dim = max(requested_dim, config.cross_head_dim) + + if ( + getattr(config, "hidden_size", None) is not None + and getattr(config, "num_attention_heads", None) is not None + ): + # For some models, num_attention_heads is a list of ints: we take the max to maximize the multiplier + num_attn_heads = getattr(config, "num_attention_heads") + num_attn_heads = num_attn_heads if isinstance(num_attn_heads, int) else max(num_attn_heads) + head_dim = head_dim if head_dim is not None else config.hidden_size // num_attn_heads + config.hidden_size *= max(requested_dim // head_dim, 1) + + if ( + getattr(config, "decoder_hidden_size", None) is not None + and getattr(config, "decoder_num_attention_heads", None) is not None + ): + decoder_head_dim = config.decoder_hidden_size // config.decoder_num_attention_heads + config.decoder_hidden_size *= max(requested_dim // decoder_head_dim, 1) + + if ( + getattr(config, "cross_hidden_size", None) is not None + and getattr(config, "cross_num_attention_heads", None) is not None + ): + cross_head_dim = ( + cross_head_dim + if cross_head_dim is not None + else config.cross_hidden_size // config.cross_num_attention_heads + ) + config.cross_hidden_size *= max(requested_dim // cross_head_dim, 1) + + # 3d rope also depends on the head dim + # (we assume easy shapes here where we get to the requested head dim at least) + if ( + getattr(config, "rope_parameters", None) is not None + and len(config.rope_parameters.get("mrope_section", [])) > 0 + ): + scaling_factor = max(requested_dim // (sum(config.rope_parameters["mrope_section"]) * 2), 1) + config.rope_parameters["mrope_section"] = [ + section * scaling_factor for section in config.rope_parameters["mrope_section"] + ] + + # Update config values + update_config_headdim(config, requested_dim) + for key in config.sub_configs: + if getattr(config, key) is not None: + sub_config = getattr(config, key) + update_config_headdim(sub_config, requested_dim) + + return config + + class GenerationTesterMixin(ExportGenerateTesterMixin): input_name = "input_ids" model_tester = None max_new_tokens = 3 + # Workaround to make it static within each mixin (model, generate) -> one override for both + _prepare_config_headdim = staticmethod(_prepare_config_headdim) + def prepare_config_and_inputs_for_generate(self, batch_size=2): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -1980,6 +2073,7 @@ def _test_attention_implementation(self, attn_implementation): "flash_attention_2": "_supports_flash_attn", "flash_attention_3": "_supports_flash_attn", "flash_attention_4": "_supports_flash_attn", + "flash_attention_torch": "_supports_flash_attn", } for model_class in self.all_generative_model_classes: @@ -1995,14 +2089,6 @@ def _test_attention_implementation(self, attn_implementation): inputs_dict[input_name] = input_data main_input = inputs_dict[model_class.main_input_name] - # FA doesn't accept masking in the middle of the sequence for now. We usually generate right-padded - # attention masks at test time and, with generate, the mask will be appended with 1s on the right, - # resulting in a mask with holes (not supported properly by FA). - if is_flash_attention_requested(requested_attention_implementation=attn_implementation): - for input_name in ("attention_mask", "decoder_attention_mask", "encoder_attention_mask"): - if input_name in inputs_dict: - inputs_dict[input_name] = torch.ones_like(inputs_dict[input_name]) - # make sure that all models have enough positions for generation if hasattr(config, "max_position_embeddings"): config.max_position_embeddings = max_new_tokens + main_input.shape[1] + 1 @@ -2010,6 +2096,25 @@ def _test_attention_implementation(self, attn_implementation): set_config_for_less_flaky_test(config) model = model_class(config) + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + + # FA doesn't accept masking in the middle of the sequence for now. We usually generate right-padded + # attention masks at test time and, with generate, the mask will be appended with 1s on the right, + # resulting in a mask with holes (not supported properly by FA). + for input_name in ("attention_mask", "decoder_attention_mask", "encoder_attention_mask"): + if input_name in inputs_dict: + inputs_dict[input_name] = torch.ones_like(inputs_dict[input_name]) + + # Some FA implementations need specific multiples at least + config = self._prepare_config_headdim(config, 16) + # If not all sub-models support flex, skip the test. We could potentially set not supported backbones # to "eager" attention, leaving it for future updates on multimodality tests sub_models_supporting_attn = [ @@ -2087,6 +2192,14 @@ def test_eager_matches_fa4_generate(self): """Tests that generate has equivalent outputs with FA4 and eager attention implementations.""" self._test_attention_implementation("flash_attention_4") + @pytest.mark.flash_attn_torch_test + @require_flash_attn_torch + @require_torch_gpu + @slow + def test_eager_matches_fa_torch_generate(self): + """Tests that generate has equivalent outputs with FA Torch and eager attention implementations.""" + self._test_attention_implementation("flash_attention_torch") + @require_flash_attn @require_torch_accelerator @pytest.mark.flash_attn_test @@ -2198,6 +2311,7 @@ def attention_mask_padding_matches_padding_free_with_position_ids( "flash_attention_2": "_supports_flash_attn", "flash_attention_3": "_supports_flash_attn", "flash_attention_4": "_supports_flash_attn", + "flash_attention_torch": "_supports_flash_attn", } for model_class in self.all_generative_model_classes: @@ -2235,6 +2349,15 @@ def attention_mask_padding_matches_padding_free_with_position_ids( ): self.skipTest(f"At least some parts of {model_class.__name__} don't support {attn_implementation}") + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + if "position_ids" not in inspect.signature(model.forward).parameters: self.skipTest("Model does not support position_ids") @@ -2361,6 +2484,22 @@ def test_flash_attention_4_padding_matches_padding_free_with_position_ids_and_fa attn_implementation="flash_attention_4", fa_kwargs=True ) + @require_flash_attn_torch + @require_torch_gpu + @pytest.mark.flash_attn_torch_test + @slow + def test_flash_attention_torch_padding_matches_padding_free_with_position_ids(self): + self.attention_mask_padding_matches_padding_free_with_position_ids(attn_implementation="flash_attention_torch") + + @require_flash_attn_torch + @require_torch_gpu + @pytest.mark.flash_attn_torch_test + @slow + def test_flash_attention_torch_padding_matches_padding_free_with_position_ids_and_fa_kwargs(self): + self.attention_mask_padding_matches_padding_free_with_position_ids( + attn_implementation="flash_attention_torch", fa_kwargs=True + ) + def _get_custom_4d_mask_test_data(self): # Sequence in which all but the last token is the same input_ids = torch.tensor( diff --git a/tests/models/blip_2/test_modeling_blip_2.py b/tests/models/blip_2/test_modeling_blip_2.py index 7b667f889986..a0b484fa8945 100644 --- a/tests/models/blip_2/test_modeling_blip_2.py +++ b/tests/models/blip_2/test_modeling_blip_2.py @@ -36,7 +36,7 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( TEST_EAGER_MATCHES_SDPA_INFERENCE_PARAMETERIZATION, @@ -69,7 +69,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/gemma3/test_modeling_gemma3.py b/tests/models/gemma3/test_modeling_gemma3.py index c288e70e33dd..5e163460f0d3 100644 --- a/tests/models/gemma3/test_modeling_gemma3.py +++ b/tests/models/gemma3/test_modeling_gemma3.py @@ -37,6 +37,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_torch, require_torch_accelerator, require_torch_gpu, @@ -347,6 +348,13 @@ def test_flash_attn_3_from_config(self): def test_flash_attn_4_from_config(self): self.flash_attn_from_config(attn_implementation="flash_attention_4", test_fwd_in_train=False) + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + def test_flash_attn_torch_from_config(self): + self.flash_attn_from_config(attn_implementation="flash_attention_torch", test_fwd_in_train=False) + def test_attention_mask_composition(self): config = self.model_tester.get_config() config.text_config._attn_implementation = "eager" diff --git a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py index a3b2df8b1bf9..ef2d52ac429b 100644 --- a/tests/models/gemma4_unified/test_modeling_gemma4_unified.py +++ b/tests/models/gemma4_unified/test_modeling_gemma4_unified.py @@ -471,6 +471,9 @@ def identity_forward(self, hidden_states): valid_fa_implementations = model._compatible_flash_implementations if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue # If we end up here, at least one model class was not skipped _has_run_at_least_one_model = True diff --git a/tests/models/instructblip/test_modeling_instructblip.py b/tests/models/instructblip/test_modeling_instructblip.py index 783a6f17f813..98a09b9ebf8f 100644 --- a/tests/models/instructblip/test_modeling_instructblip.py +++ b/tests/models/instructblip/test_modeling_instructblip.py @@ -45,7 +45,7 @@ ) from transformers.utils import is_torch_available, is_vision_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -67,7 +67,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py index 85035e6ece32..582f4bc30255 100644 --- a/tests/models/instructblipvideo/test_modeling_instructblipvideo.py +++ b/tests/models/instructblipvideo/test_modeling_instructblipvideo.py @@ -43,7 +43,7 @@ ) from transformers.utils import is_torch_available -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ( ModelTesterMixin, @@ -65,7 +65,7 @@ def _prepare_qformer_config_headdim(config, requested_dim): - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) + config = _prepare_config_headdim(config, requested_dim) config.qformer_config.encoder_hidden_size = config.vision_config.hidden_size return config diff --git a/tests/models/janus/test_modeling_janus.py b/tests/models/janus/test_modeling_janus.py index 595b7ff713c6..87b2b3aa165d 100644 --- a/tests/models/janus/test_modeling_janus.py +++ b/tests/models/janus/test_modeling_janus.py @@ -41,7 +41,7 @@ torch_device, ) -from ...generation.test_utils import GenerationTesterMixin +from ...generation.test_utils import GenerationTesterMixin, _prepare_config_headdim from ...test_configuration_common import ConfigTester from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor from ...test_pipeline_mixin import PipelineTesterMixin @@ -55,6 +55,14 @@ from PIL import Image +def _prepare_janus_config_headdim(config, requested_dim): + config = _prepare_config_headdim(config, requested_dim) + # Sync projection_dim with text hidden_size since aligner output must match text embeddings + if hasattr(config, "vision_config") and hasattr(config, "text_config"): + config.vision_config.projection_dim = config.text_config.hidden_size + return config + + class JanusVisionText2TextModelTester: def __init__( self, @@ -206,13 +214,7 @@ def _prepare_config_headdim(config, requested_dim): Override to ensure vision_config.projection_dim stays in sync with text_config.hidden_size. The aligner projects vision features to text embedding dimension, so they must match. """ - from tests.test_modeling_common import ModelTesterMixin - - config = ModelTesterMixin._prepare_config_headdim(config, requested_dim) - # Sync projection_dim with text hidden_size since aligner output must match text embeddings - if hasattr(config, "vision_config") and hasattr(config, "text_config"): - config.vision_config.projection_dim = config.text_config.hidden_size - return config + return _prepare_janus_config_headdim(config, requested_dim) def setUp(self): self.model_tester = JanusVisionText2TextModelTester(self) diff --git a/tests/models/sam3/test_modeling_sam3.py b/tests/models/sam3/test_modeling_sam3.py index 881d55f3cc42..6921434bda80 100644 --- a/tests/models/sam3/test_modeling_sam3.py +++ b/tests/models/sam3/test_modeling_sam3.py @@ -640,6 +640,20 @@ def test_flash_attn_4_inference_equivalence(self): def test_flash_attn_4_inference_equivalence_right_padding(self): pass + @unittest.skip( + reason="Sam3Model creates attention masks from features (with gradients), " + "which is incompatible with flash attention's expectation of binary masks" + ) + def test_flash_attn_torch_inference_equivalence(self): + pass + + @unittest.skip( + reason="Sam3Model creates attention masks from features (with gradients), " + "which is incompatible with flash attention's expectation of binary masks" + ) + def test_flash_attn_torch_inference_equivalence_right_padding(self): + pass + @unittest.skip( reason="Sam3Model creates attention masks from features (with gradients), " "which is incompatible with flash attention's expectation of binary masks" diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py index 2c9a48d4c884..df7890cf32c8 100644 --- a/tests/test_modeling_common.py +++ b/tests/test_modeling_common.py @@ -98,6 +98,7 @@ require_flash_attn, require_flash_attn_3, require_flash_attn_4, + require_flash_attn_torch, require_kernels, require_non_hpu, require_torch, @@ -121,10 +122,11 @@ is_torch_bf16_available_on_device, is_torch_fp16_available_on_device, ) +from transformers.utils.generic import is_flash_attention_requested from transformers.utils.output_capturing import CompileableContextVar, OutputRecorder from .exporters.test_export import ExportTesterMixin -from .generation.test_utils import GenerationTesterMixin +from .generation.test_utils import GenerationTesterMixin, _prepare_config_headdim if is_torch_available(): @@ -742,6 +744,9 @@ class ModelTesterMixin(ExportTesterMixin): _is_composite = False model_split_percents = [0.5, 0.7, 0.9] + # Workaround to make it static within each mixin (model, generate) -> one override for both + _prepare_config_headdim = staticmethod(_prepare_config_headdim) + # Note: for all mixins that utilize the Hub in some way, we should ensure that # they contain the `hub_retry` decorator in case of failures. def __init_subclass__(cls, **kwargs): @@ -3551,6 +3556,9 @@ def flash_attn_inference_equivalence( valid_fa_implementations = model._compatible_flash_implementations if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue # If we end up here, at least one model class was not skipped _has_run_at_least_one_model = True @@ -3731,6 +3739,22 @@ def test_flash_attn_4_inference_equivalence(self): def test_flash_attn_4_inference_equivalence_right_padding(self): self.flash_attn_inference_equivalence(attn_implementation="flash_attention_4", padding_side="right") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + @is_flaky() + def test_flash_attn_torch_inference_equivalence(self): + self.flash_attn_inference_equivalence(attn_implementation="flash_attention_torch", padding_side="left") + + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + @is_flaky() + def test_flash_attn_torch_inference_equivalence_right_padding(self): + self.flash_attn_inference_equivalence(attn_implementation="flash_attention_torch", padding_side="right") + def test_attn_implementation_composite_models(self): """ Tests if composite models can receive a dict object as attn_implementation, where each key should be @@ -4157,6 +4181,12 @@ def test_flash_attn_3_can_dispatch_composite_models(self): def test_flash_attn_4_can_dispatch_composite_models(self): self.flash_attn_can_dispatch_composite_models(attn_implementation="flash_attention_4") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + def test_flash_attn_torch_can_dispatch_composite_models(self): + self.flash_attn_can_dispatch_composite_models(attn_implementation="flash_attention_torch") + @require_flash_attn @require_torch_accelerator @require_bitsandbytes @@ -4269,12 +4299,24 @@ def flash_attn_from_config(self, attn_implementation: str, test_fwd_in_train: bo self.skipTest(f"{model_class.__name__} does not support {attn_implementation}") config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # Some FA implementations need specific multiples at least + config = self._prepare_config_headdim(config, 16) + model = model_class(config) # let's construct it here to see if any submodels can't support flash attn if not all( submodel._supports_flash_attn for submodel in model.modules() if isinstance(submodel, PreTrainedModel) ): self.skipTest(reason=f"At least some parts of this model do not support {attn_implementation}") + if is_flash_attention_requested(requested_attention_implementation=attn_implementation): + # Check for validity of the FA implementation, some are not compatible + valid_fa_implementations = model._compatible_flash_implementations + if valid_fa_implementations is not None and attn_implementation not in valid_fa_implementations: + continue + invalid_fa_implementations = model._incompatible_flash_implementations + if invalid_fa_implementations is not None and attn_implementation in invalid_fa_implementations: + continue + # TODO: to change it in the future with other relevant auto classes fa_model = model_class._from_config( config, attn_implementation=attn_implementation, dtype=torch.bfloat16 @@ -4331,6 +4373,13 @@ def test_flash_attn_3_from_config(self): def test_flash_attn_4_from_config(self): self.flash_attn_from_config(attn_implementation="flash_attention_4") + @require_flash_attn_torch + @require_torch_gpu + @mark.flash_attn_torch_test + @slow + def test_flash_attn_torch_from_config(self): + self.flash_attn_from_config(attn_implementation="flash_attention_torch") + def test_sliding_window_mask(self): """Tests that we can control the sliding window attention behavior of a model.""" config, inputs = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/utils/test_modeling_utils.py b/tests/utils/test_modeling_utils.py index d01002e8cc1e..6676d97f063f 100644 --- a/tests/utils/test_modeling_utils.py +++ b/tests/utils/test_modeling_utils.py @@ -59,7 +59,6 @@ is_torch_available, logging, ) -from transformers.modeling_flash_attention_utils import is_flash_attn_available from transformers.models.mistral.modeling_mistral import MistralModel from transformers.testing_utils import ( TOKEN, @@ -94,6 +93,7 @@ is_flash_attn_2_available, is_flash_attn_3_available, is_flash_attn_4_available, + is_flash_attn_torch_available, is_kernels_available, is_torch_npu_available, ) @@ -767,7 +767,7 @@ def test_model_from_pretrained_attn_implementation(self): # 2. explicit from_pretrained's attn_implementation argument with a config argument attn_implementation_available = ["eager", "sdpa"] - if is_flash_attn_available(): + if is_flash_attn_2_available(): attn_implementation_available.append("flash_attention_2") if is_flash_attn_3_available(): @@ -776,6 +776,9 @@ def test_model_from_pretrained_attn_implementation(self): if is_flash_attn_4_available(): attn_implementation_available.append("flash_attention_4") + if is_flash_attn_torch_available(): + attn_implementation_available.append("flash_attention_torch") + for requested_attn_implementation in attn_implementation_available: model = AutoModelForCausalLM.from_pretrained( TINY_MISTRAL, attn_implementation=requested_attn_implementation @@ -795,7 +798,7 @@ def test_model_from_config_attn_implementation(self): # 3. config created with explicit attn_implementation and from_config overriding with explicit attn_implementation argument attn_implementation_available = ["eager", "sdpa"] - if is_flash_attn_available(): + if is_flash_attn_2_available(): attn_implementation_available.append("flash_attention_2") if is_flash_attn_3_available(): @@ -804,6 +807,9 @@ def test_model_from_config_attn_implementation(self): if is_flash_attn_4_available(): attn_implementation_available.append("flash_attention_4") + if is_flash_attn_torch_available(): + attn_implementation_available.append("flash_attention_torch") + for requested_attn_implementation in attn_implementation_available: config = AutoConfig.from_pretrained(TINY_MISTRAL, attn_implementation=requested_attn_implementation) # Ensure the config was set correctly @@ -3109,7 +3115,7 @@ def test_not_available_flash(self): _ = AutoModel.from_pretrained( "hf-internal-testing/tiny-random-GPTBigCodeModel", attn_implementation="flash_attention_2" ) - self.assertTrue("the package for FlashAttention2 doesn't seem to be installed." in str(cm.exception)) + self.assertTrue("the package for Flash Attention 2 doesn't seem to be installed." in str(cm.exception)) @parameterized.expand( [ @@ -3157,6 +3163,7 @@ def test_flash_attn_available_no_keyerror_when_missing_from_distribution_map(sel self.assertFalse(is_flash_attn_2_available()) self.assertFalse(is_flash_attn_3_available()) self.assertFalse(is_flash_attn_4_available()) + self.assertFalse(is_flash_attn_torch_available()) def test_not_available_flash_with_config(self): if is_flash_attn_2_available(): @@ -3179,7 +3186,7 @@ def test_not_available_flash_with_config(self): attn_implementation="flash_attention_2", ) - self.assertTrue("the package for FlashAttention2 doesn't seem to be installed." in str(cm.exception)) + self.assertTrue("the package for Flash Attention 2 doesn't seem to be installed." in str(cm.exception)) def test_kernels_fallback(self): if not is_kernels_available():