From 83df8135221706bf76b315e6892a32ce096cc2a8 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 11:00:57 +0800 Subject: [PATCH 01/11] Adds flash_dmattn availability check Introduces a cached utility to detect flash_dmattn support and minimum version on GPU setups. Validates package presence and requires >= 1.0.0 under CUDA; reports unavailable for HIP/MLU. Enables safer conditional feature use and avoids runtime errors on unsupported platforms. --- src/transformers/utils/import_utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index 1cfbb0c9b88e..f6770492d0ea 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -868,6 +868,24 @@ def is_flash_attn_greater_or_equal(library_version: str) -> bool: return is_available and version.parse(flash_attn_version) >= version.parse(library_version) +@lru_cache +def is_flash_dmattn_available(): + is_available, flash_attn_version = _is_package_available("flash_dmattn", return_version=True) + if not is_available or not (is_torch_cuda_available() or is_torch_mlu_available()): + return False + + import torch + + if torch.version.cuda: + return version.parse(flash_attn_version) >= version.parse("1.0.0") + elif torch.version.hip: + return False + elif is_torch_mlu_available(): + return False + else: + return False + + @lru_cache def is_huggingface_hub_greater_or_equal(library_version: str, accept_dev: bool = False) -> bool: is_available, hub_version = _is_package_available("huggingface_hub", return_version=True) From ac44a5582180fdecb6549816df2804976ee81ec9 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 11:01:32 +0800 Subject: [PATCH 02/11] Adds flash dmattn availability check export Exposes the new availability helper to match other flash attention checks. Enables downstream feature gating and conditional logic based on backend presence. --- src/transformers/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 44589d96c841..9ce78f3f2979 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -144,6 +144,7 @@ is_flash_attn_3_available, is_flash_attn_greater_or_equal, is_flash_attn_greater_or_equal_2_10, + is_flash_dmattn_available, is_flute_available, is_fp_quant_available, is_fsdp_available, From ef0d9439bd8b1a84b5c3efc42e57db0c170d4263 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 11:02:14 +0800 Subject: [PATCH 03/11] Add test gate for Flash Dynamic Mask Attention Adds a decorator to conditionally skip tests that depend on Flash Dynamic Mask Attention when the feature or kernels backend is unavailable. Verifies the kernels registry for the dynamic mask attention kernel to prevent spurious failures and aligns gating with existing Flash Attention tests. --- src/transformers/testing_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 81f4de255f9c..e55d49972fe1 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -88,6 +88,7 @@ is_fbgemm_gpu_available, is_flash_attn_2_available, is_flash_attn_3_available, + is_flash_dmattn_available, is_flute_available, is_fp_quant_available, is_fsdp_available, @@ -599,6 +600,25 @@ def require_flash_attn(test_case): return unittest.skipUnless(kernels_available | flash_attn_available, "test requires Flash Attention")(test_case) +def require_flash_dmattn(test_case): + """ + Decorator marking a test that requires Flash Dynamic Mask Attention. + + These tests are skipped when Flash Dynamic Mask Attention isn't installed. + + """ + flash_dmattn_available = is_flash_dmattn_available() + kernels_available = is_kernels_available() + try: + from kernels import get_kernel + + get_kernel("kernels-community/flash-dmattn") + except Exception as _: + kernels_available = False + + return unittest.skipUnless(kernels_available | flash_dmattn_available, "test requires Flash Dynamic Mask Attention")(test_case) + + def require_kernels(test_case): """ Decorator marking a test that requires the kernels library. From d10025580e8f826bf711ab7f31dfb6f7f3c1d8ba Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 11:42:37 +0800 Subject: [PATCH 04/11] Adds Flash Dynamic Mask Attention integration Introduces a forward wrapper for the flash dynamic mask attention path to enable the optimized backend with correct input layout and dtype handling. Ensures stable dtype under autocast, quantization, and PEFT LayerNorm casting, validates non-empty inputs, respects an is_causal override, and warns when attention outputs are requested but unsupported. --- .../flash_dynamic_mask_attention.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/transformers/integrations/flash_dynamic_mask_attention.py diff --git a/src/transformers/integrations/flash_dynamic_mask_attention.py b/src/transformers/integrations/flash_dynamic_mask_attention.py new file mode 100644 index 000000000000..961b0e81e78f --- /dev/null +++ b/src/transformers/integrations/flash_dynamic_mask_attention.py @@ -0,0 +1,83 @@ +from typing import Optional + +import torch + +from ..modeling_flash_dynamic_mask_attention_utils import _flash_dynamic_mask_attention_forward +from ..utils import logging + + +logger = logging.get_logger(__name__) + + +def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtype: + """If the query is in float32, return a target dtype compatible with flash dynamic mask attention. Return None otherwise.""" + if query.dtype == torch.float32: + if torch.is_autocast_enabled(): + return torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(module.config, "_pre_quantization_dtype"): + return module.config._pre_quantization_dtype + else: + return next(layer for layer in module.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype + return None + + +def flash_dynamic_mask_attention_forward( + module: torch.nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: Optional[torch.Tensor], + attention_bias: Optional[torch.Tensor], + scaling: Optional[float] = None, + softcap: Optional[float] = None, + is_causal: Optional[bool] = None, + **kwargs, +) -> tuple[torch.Tensor, None]: + if kwargs.get("output_attentions", False): + logger.warning_once( + "`flash_dynamic_mask_attention` does not support `output_attentions=True`." + " Please set your attention to `eager` if you want any of these features." + ) + + # This is before the transpose + seq_len = query.shape[2] + + if any(dim == 0 for dim in query.shape): + raise ValueError( + "Tensor query has shape with a zero dimension.\n" + "FlashDynamicMaskAttention does not support inputs with dim=0.\n" + "Please check your input shapes or use SDPA instead." + ) + # FDMA uses non-transposed inputs + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (usually our RMSNorm modules handle it correctly) + target_dtype = get_target_dtype(query, module) + + # Instead of relying on the value set in the module directly, we use the is_causal passed in kwargs if it is presented + is_causal = is_causal if is_causal is not None else module.is_causal + + attn_output = _flash_dynamic_mask_attention_forward( + query, + key, + value, + attention_mask, + attention_bias, + query_length=seq_len, + is_causal=is_causal, + softmax_scale=scaling, + softcap=softcap, + target_dtype=target_dtype, + implementation=module.config._attn_implementation, + layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None, + **kwargs, + ) + + return attn_output, None From a5ba48c12de2e06cd422dcd33e0682c31603f4bc Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 12:22:12 +0800 Subject: [PATCH 05/11] Adds flash dynamic mask attention utilities Introduces a lazy-loaded wrapper for flash dynamic mask attention to enable optional use and compatibility across versions and torch.compile modes. Processes and filters kwargs against kernel capabilities, exposing softmax scale, softcap, deterministic (env-controlled), and attention sink auxiliary when supported. Mitigates PEFT-induced fp32 casts by restoring target dtypes for queries/keys/values and bias to ensure kernel compatibility. Handles causal vs full attention, applies attention mask and bias, and normalizes outputs when kernels return tuples. Improves performance, stability, and configurability while keeping the API consistent. --- ...ling_flash_dynamic_mask_attention_utils.py | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 src/transformers/modeling_flash_dynamic_mask_attention_utils.py diff --git a/src/transformers/modeling_flash_dynamic_mask_attention_utils.py b/src/transformers/modeling_flash_dynamic_mask_attention_utils.py new file mode 100644 index 000000000000..c00e21db2868 --- /dev/null +++ b/src/transformers/modeling_flash_dynamic_mask_attention_utils.py @@ -0,0 +1,240 @@ +# Copyright 2025 Jingze Shi and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import os +from functools import partial +from typing import Optional + +import torch +import torch.nn.functional as F + +from .utils import ( + is_flash_dmattn_available, + logging, +) + + +logger = logging.get_logger(__name__) + + +# `globals()` is not compatible with dynamo, hence we have do define them in global scope ourselves +_fdma_fn = None + +# function that processes kwargs, generalized to handle any supported kwarg within the function +_process_flash_kwargs_fn = None +# exceptions where hf API doesn't match the original FDMA API +_hf_api_to_flash_mapping = { + "dropout": None, + "sliding_window": None, +} + + +def _lazy_imports(implementation: Optional[str]): + """ + Lazy loads the respective flash dynamic mask attention implementations. + + Return: + flash_dmattn_func: The base flash dynamic mask attention function. + """ + is_fdma = is_flash_dmattn_available() + + if (implementation == "flash_dynamic_mask_attention" and is_fdma) or (implementation is None and is_fdma): + from flash_dmattn import flash_dmattn_func + + return flash_dmattn_func + + +def _lazy_define_process_function(flash_function): + """ + Depending on the version and kernel some features are not supported. Due to limitations in + `torch.compile`, we opt to statically type which (optional) kwarg parameters are supported + within `_process_flash_dynamic_mask_attention_kwargs`. + + NOTE: While all supported kwargs are marked as `True`, everything else is marked as `False`. + This might be confusing for kwargs that we use in any case, e.g. `is_causal`. + """ + + flash_parameters = inspect.signature(flash_function).parameters + process_parameters = inspect.signature(_process_flash_dynamic_mask_attention_kwargs).parameters + + supports_mapping = {} + for param in process_parameters: + fdma_param = _hf_api_to_flash_mapping.get(param, param) + supports_mapping[fdma_param] = fdma_param in flash_parameters + + return partial(_process_flash_dynamic_mask_attention_kwargs, supports_mapping=supports_mapping) + + +def lazy_import_flash_dynamic_mask_attention(implementation: Optional[str], force_import: Optional[bool] = False): + """ + Lazily import flash dmattn and return the respective functions + flags. + + NOTE: For fullgraph, this needs to be called before compile, while no fullgraph can + work without preloading. See `load_and_register_kernel` in `integrations.hub_kernels`. + """ + global _fdma_fn + if force_import or any(k is None for k in [_fdma_fn]): + _fdma_fn = _lazy_imports(implementation) + + global _process_flash_kwargs_fn + if force_import or _process_flash_kwargs_fn is None: + _process_flash_kwargs_fn = _lazy_define_process_function(_fdma_fn) + + return (_fdma_fn), _process_flash_kwargs_fn + + +def fdma_peft_integration_check( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + bias: Optional[torch.Tensor], + target_dtype: Optional[torch.dtype] = None +): + """ + PEFT usually casts the layer norms in float32 for training stability reasons + therefore the input hidden states gets silently casted in float32. Hence, we need + cast them back in float16 / bfloat16 just to be sure everything works as expected. + This might slowdown training & inference so it is recommended to not cast the LayerNorms! + """ + if target_dtype and q.dtype == torch.float32: + logger.warning_once(f"Casting fp32 inputs back to {target_dtype} for flash-dmattn compatibility.") + q, k, v = q.to(target_dtype), k.to(target_dtype), v.to(target_dtype) + if bias is not None: + bias = bias.to(target_dtype) + return q, k, v, bias + + +def _process_flash_dynamic_mask_attention_kwargs( + query_length: int, + key_length: int, + is_causal: bool, + softmax_scale: Optional[float] = None, + softcap: Optional[float] = None, + deterministic: Optional[bool] = None, + s_aux: Optional[torch.Tensor] = None, + supports_mapping: Optional[dict[str, bool]] = None, + **kwargs, +): + """ + Returns a set of kwargs that are passed down to the according flash dynamic mask attention function based on + requested features and whether it is supported - depends on the version and kernel implementation + which is dynamically configured at `lazy_import_flash_dynamic_mask_attention`. The (un)supported features can be + inspected in `supports_mapping`, see `_lazy_define_process_function` for more details. + + Args: + query_length (`int`): + Length of the query states + key_length (`int`): + Length of the key states + is_causal (`bool`): + Whether we perform causal (decoder) attention or full attention. + softmax_scale (`float`, *optional*): + The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`. + softcap (`float`, *optional*): + Softcap for the attention logits, used e.g. in gemma2. + deterministic (`bool`, *optional*): + Determines if the deterministic option introduced in flash_dmattn>=1.0.0 is enabled. + s_aux (`torch.Tensor`, *optional*): + Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head. + Return: + flash_kwargs (`dict`): + A dict of kwargs that are requested and supported. + """ + flash_kwargs = { + "is_causal": is_causal and not query_length == 1, + "softmax_scale": softmax_scale, + } + + if supports_mapping["deterministic"]: + flash_kwargs["deterministic"] = ( + deterministic if deterministic is not None else os.getenv("FLASH_DMATTN_DETERMINISTIC", "0") == "1" + ) + + if supports_mapping["softcap"] and softcap is not None: + flash_kwargs["softcap"] = softcap + + # Only within kernel implementation atm + if supports_mapping["s_aux"] and s_aux is not None: + flash_kwargs["s_aux"] = s_aux + + return flash_kwargs + + +def _flash_dynamic_mask_attention_forward( + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + attention_mask: Optional[torch.Tensor], + attention_bias: Optional[torch.Tensor], + query_length: int, + is_causal: bool, + softmax_scale: Optional[float] = None, + softcap: Optional[float] = None, + deterministic: Optional[bool] = None, + target_dtype: Optional[torch.dtype] = None, + implementation: Optional[str] = None, + **kwargs, +): + """ + Calls the forward method of Flash Dynamic Mask Attention - if the input hidden states contain at least one padding token + first unpad the input, then computes the attention scores and pad the final attention scores. + + (Optional) kwargs are described further in `_process_flash_dynamic_mask_attention_kwargs` and `FlashDynamicMaskAttentionKwargs`. + + Args: + query_states (`torch.Tensor`): + Input query states to be passed to Flash Dynamic Mask Attention API + key_states (`torch.Tensor`): + Input key states to be passed to Flash Dynamic Mask Attention API + value_states (`torch.Tensor`): + Input value states to be passed to Flash Dynamic Mask Attention API + attention_mask (`torch.Tensor`, *optional*): + The padding mask - corresponds to a tensor of size `(batch_size, num_heads, query_len, key_len)` where 0 stands for the + position of padding tokens and 1 for the position of non-padding tokens. + attention_bias (`torch.Tensor`, *optional*): + The attention bias tensor of size `(batch_size, num_heads, query_len, key_len)` to add to attention scores. + implementation (`str`, *optional*): + The attention implementation to use. If None, will default to the one based on the environment. + """ + + (fdma_fn), process_flash_kwargs_fn = lazy_import_flash_dynamic_mask_attention(implementation) + + # PEFT possibly silently casts tensors to fp32, this potentially reconverts to correct dtype or is a no op + query_states, key_states, value_states, attention_bias = fdma_peft_integration_check( + query_states, key_states, value_states, attention_bias, target_dtype + ) + + # Extract the flash dynamic mask attention kwargs that have been requested (and are supported by the implementation) + flash_kwargs = process_flash_kwargs_fn( + query_length=query_length, + key_length=key_states.size(1), + is_causal=is_causal, + softmax_scale=softmax_scale, + softcap=softcap, + deterministic=deterministic, + **kwargs, + ) + + out = fdma_fn( + query_states, + key_states, + value_states, + attention_mask, + attention_bias, + **flash_kwargs, + ) + if isinstance(out, tuple): + out = out[0] + + return out From 0cca929c7d26bc4916aecb83d6b39b0c9dd25902 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 12:24:54 +0800 Subject: [PATCH 06/11] Add Flash Dynamic Mask Attention support Enables a new high-performance attention backend with dynamic masking and wires it into dispatch and the attention interface. Adds a capability flag and a comprehensive availability check covering package/version, device, and dtype, with NPU handling and clearer user guidance. Improves early, actionable errors and warnings. Updates help text to advertise the new attn_implementation option. --- src/transformers/modeling_utils.py | 94 ++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 794c238c0fa7..96f316eeac8e 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -61,6 +61,7 @@ from .integrations.deepspeed import _load_state_dict_into_zero3_model from .integrations.eager_paged import eager_paged_attention_forward from .integrations.flash_attention import flash_attention_forward +from .integrations.flash_dynamic_mask_attention import flash_dynamic_mask_attention_forward from .integrations.flash_paged import paged_attention_forward from .integrations.flex_attention import flex_attention_forward from .integrations.hub_kernels import is_kernel, load_and_register_attn_kernel @@ -104,6 +105,7 @@ is_bitsandbytes_available, is_flash_attn_2_available, is_flash_attn_3_available, + is_flash_dmattn_available, is_kernels_available, is_offline_mode, is_remote_url, @@ -1696,6 +1698,9 @@ class PreTrainedModel(nn.Module, EmbeddingAccessMixin, ModuleUtilsMixin, PushToH # Flash Attention support _supports_flash_attn = False + # Flash Dynamic Mask Attention support + _supports_flash_dmattn = False + # SDPA support _supports_sdpa = False @@ -2292,6 +2297,90 @@ def _flash_attn_3_can_dispatch(self, is_init_check: bool = False) -> bool: return True + def _flash_dmattn_can_dispatch(self, is_init_check: bool = False) -> bool: + """ + Check the availability of Flash Dynamic Mask Attention for a given model. + + Args: + is_init_check (`bool`, *optional*): + Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are + fully instantiated. This is needed as we also check the devices of the weights, which are only available + later after __init__. This allows to raise proper exceptions early before instantiating the full models + if we know that the model does not support the requested attention. + """ + dtype = self.config.dtype + + # check `supports_flash_dmattn` for BC with custom code. TODO: remove after a few releases + if not (self._supports_flash_dmattn or getattr(self, "_supports_flash_dmattn", False)): + raise ValueError( + f"{self.__class__.__name__} does not support Flash Dynamic Mask Attention yet. Please request to add support where" + f" the model is hosted, on its model hub page: https://huggingface.co/{self.config._name_or_path}/discussions/new" + " or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/new" + ) + + if not is_flash_dmattn_available(): + preface = "FlashDynamicMaskAttention has been toggled on, but it cannot be used due to the following error:" + install_message = "Please refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashdynamicmaskattention to install Flash Dynamic Mask Attention." + + # package `flash-dmattn` can not be installed on Ascend NPU, following validation logics can be ignored. + if is_torch_npu_available(): + logger.info("Detect using FlashDynamicMaskAttention on Ascend NPU.") + return True + + if importlib.util.find_spec("flash_dmattn") is None: + raise ImportError(f"{preface} the package flash_dmattn seems to be not installed. {install_message}") + else: + # Check FDMA installed version compatibility + flash_attention_version = version.parse(importlib.metadata.version("flash_dmattn")) + if torch.version.cuda: + if flash_attention_version < version.parse("1.0.0"): + raise ImportError( + f"{preface} you need flash_dmattn package version to be greater or equal than 1.0.0. Detected version {flash_attention_version}. {install_message}" + ) + elif not torch.cuda.is_available(): + raise ValueError( + f"{preface} Flash Dynamic Mask Attention is not available on CPU. Please make sure torch can access a CUDA device." + ) + else: + raise ImportError(f"{preface} Flash Dynamic Mask Attention is not available. {install_message}") + elif torch.version.hip: + raise ImportError(f"{preface} Flash Dynamic Mask Attention is not available. {install_message}") + + if dtype is None: + logger.warning_once( + "You are attempting to use Flash Dynamic Mask Attention without specifying a torch dtype. This might lead to unexpected behaviour" + ) + elif dtype is not None and dtype not in [torch.float16, torch.bfloat16]: + logger.warning_once( + "Flash Dynamic Mask Attention only supports torch.float16 and torch.bfloat16 dtypes, but" + f" the current dype in {self.__class__.__name__} is {dtype}. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator," + ' or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_dynamic_mask_attention", dtype=torch.float16)`' + ) + + # With the early check, the parameters are not yet initialized correctly + if not is_init_check: + param_devices = list({param.device for param in self.parameters()}) + if len(param_devices) == 1 and param_devices[0].type == "cpu": + if torch.cuda.is_available(): + logger.warning_once( + "You are attempting to use Flash Dynamic Mask Attention with a model not initialized on GPU. Make sure to move the model to GPU" + " after initializing it on CPU with `model.to('cuda')`." + ) + elif is_torch_mlu_available(): + logger.warning_once( + "You are attempting to use Flash Dynamic Mask Attention with a model not initialized on MLU. Make sure to move the model to MLU" + " after initializing it on CPU with `model.to('mlu')`." + ) + else: + raise ValueError( + "You are attempting to use Flash Dynamic Mask Attention with a model not initialized on GPU and with no GPU available. " + "This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map " + "or initialising the model on CPU and then moving it to GPU." + ) + + # If no error raise by this point, we can return `True` + return True + def _sdpa_can_dispatch(self, is_init_check: bool = False) -> bool: """ Check the availability of SDPA for a given model. @@ -2423,6 +2512,8 @@ def get_correct_attn_implementation(self, requested_attention: Optional[str], is # check `supports_flash_attn_2` for BC with custom code. TODO: remove after a few releases if self._supports_flash_attn or getattr(self, "_supports_flash_attn_2", False): message += ', `"attn_implementation=flash_attention_3"`, `"attn_implementation=flash_attention_2"`, `"attn_implementation=paged|flash_attention_2"`' + if self._supports_flash_dmattn: + message += ', `"attn_implementation=flash_dynamic_mask_attention"`' if self._supports_sdpa: message += ', `"attn_implementation=sdpa"`, `"attn_implementation=paged|spda"`' if self._supports_flex_attn: @@ -2434,6 +2525,8 @@ def get_correct_attn_implementation(self, requested_attention: Optional[str], is self._flash_attn_2_can_dispatch(is_init_check) elif "flash_attention_3" in applicable_attention: self._flash_attn_3_can_dispatch(is_init_check) + elif "flash_dynamic_mask_attention" in applicable_attention: + self._flash_dmattn_can_dispatch(is_init_check) elif "flex_attention" in applicable_attention: self._flex_attn_can_dispatch(is_init_check) elif "sdpa" in applicable_attention: @@ -5386,6 +5479,7 @@ class AttentionInterface(GeneralInterface): _global_mapping = { "flash_attention_3": flash_attention_forward, "flash_attention_2": flash_attention_forward, + "flash_dynamic_mask_attention": flash_dynamic_mask_attention_forward, "flex_attention": flex_attention_forward, "sdpa": sdpa_attention_forward, "paged|flash_attention_2": paged_attention_forward, From 4cd7e5654227f6394acf8589016e5b1dba4c11d9 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 14:43:42 +0800 Subject: [PATCH 07/11] Adds flash_dmattn to attention backends docs Documents availability of the flash_dmattn attention implementation alongside existing options. Improves clarity and discoverability by linking to the project. --- docs/source/en/attention_interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/attention_interface.md b/docs/source/en/attention_interface.md index 621aa7409da0..45f52bdd5503 100644 --- a/docs/source/en/attention_interface.md +++ b/docs/source/en/attention_interface.md @@ -22,7 +22,7 @@ supported models. Most recent models can now switch from one attention function used in the Attention layer to the other, thanks to a simple mapping. By default, we provide the implementation for [`sdpa`](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html), -[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention) +[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention), [`flash_dmattn`](https://github.com/SmallDoges/flash-dmattn) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention) as well as `eager`, which is a simple matrix multiplication without any optimization on top. This is the setting you can usually choose when instantiating a model: From 5d8cba21bd3ce78f82da848310e39c9aec944a2a Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 14:44:13 +0800 Subject: [PATCH 08/11] Removes unused import and simplifies causal check Removes an unused import to reduce noise and satisfy linters. Simplifies the causal condition for readability without changing behavior. Adds a trailing comma in parameters to align with formatting conventions and cleaner diffs. --- .../modeling_flash_dynamic_mask_attention_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_flash_dynamic_mask_attention_utils.py b/src/transformers/modeling_flash_dynamic_mask_attention_utils.py index c00e21db2868..66cd27e75e98 100644 --- a/src/transformers/modeling_flash_dynamic_mask_attention_utils.py +++ b/src/transformers/modeling_flash_dynamic_mask_attention_utils.py @@ -17,7 +17,6 @@ from typing import Optional import torch -import torch.nn.functional as F from .utils import ( is_flash_dmattn_available, @@ -99,7 +98,7 @@ def fdma_peft_integration_check( k: torch.Tensor, v: torch.Tensor, bias: Optional[torch.Tensor], - target_dtype: Optional[torch.dtype] = None + target_dtype: Optional[torch.dtype] = None, ): """ PEFT usually casts the layer norms in float32 for training stability reasons @@ -152,7 +151,7 @@ def _process_flash_dynamic_mask_attention_kwargs( A dict of kwargs that are requested and supported. """ flash_kwargs = { - "is_causal": is_causal and not query_length == 1, + "is_causal": is_causal and query_length != 1, "softmax_scale": softmax_scale, } From 4e9d9789b34f3fe7f5faef8f574589717e634aa4 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 14:45:59 +0800 Subject: [PATCH 09/11] Reformat Flash Dynamic Mask Attention availability error messages for readability --- src/transformers/modeling_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 96f316eeac8e..8ac48b90b039 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2319,7 +2319,9 @@ def _flash_dmattn_can_dispatch(self, is_init_check: bool = False) -> bool: ) if not is_flash_dmattn_available(): - preface = "FlashDynamicMaskAttention has been toggled on, but it cannot be used due to the following error:" + preface = ( + "FlashDynamicMaskAttention has been toggled on, but it cannot be used due to the following error:" + ) install_message = "Please refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashdynamicmaskattention to install Flash Dynamic Mask Attention." # package `flash-dmattn` can not be installed on Ascend NPU, following validation logics can be ignored. @@ -2342,7 +2344,9 @@ def _flash_dmattn_can_dispatch(self, is_init_check: bool = False) -> bool: f"{preface} Flash Dynamic Mask Attention is not available on CPU. Please make sure torch can access a CUDA device." ) else: - raise ImportError(f"{preface} Flash Dynamic Mask Attention is not available. {install_message}") + raise ImportError( + f"{preface} Flash Dynamic Mask Attention is not available. {install_message}" + ) elif torch.version.hip: raise ImportError(f"{preface} Flash Dynamic Mask Attention is not available. {install_message}") From 50e3b1d24dfc435bf2ad83a042922ce0c37603c8 Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 14:46:23 +0800 Subject: [PATCH 10/11] Reformat require_flash_dmattn skipUnless call for readability --- src/transformers/testing_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index e55d49972fe1..2599f0f64034 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -616,7 +616,9 @@ def require_flash_dmattn(test_case): except Exception as _: kernels_available = False - return unittest.skipUnless(kernels_available | flash_dmattn_available, "test requires Flash Dynamic Mask Attention")(test_case) + return unittest.skipUnless( + kernels_available | flash_dmattn_available, "test requires Flash Dynamic Mask Attention" + )(test_case) def require_kernels(test_case): From ed0faf6ae37ca22edf5b5576a072142e931ac66a Mon Sep 17 00:00:00 2001 From: LoserCheems Date: Mon, 27 Oct 2025 14:46:41 +0800 Subject: [PATCH 11/11] Rename flash_dmattn to flash_dynamic_mask_attention in attention_interface.md --- docs/source/en/attention_interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/attention_interface.md b/docs/source/en/attention_interface.md index 45f52bdd5503..30ee57be7dde 100644 --- a/docs/source/en/attention_interface.md +++ b/docs/source/en/attention_interface.md @@ -22,7 +22,7 @@ supported models. Most recent models can now switch from one attention function used in the Attention layer to the other, thanks to a simple mapping. By default, we provide the implementation for [`sdpa`](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html), -[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention), [`flash_dmattn`](https://github.com/SmallDoges/flash-dmattn) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention) +[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention), [`flash_dynamic_mask_attention`](https://github.com/SmallDoges/flash-dmattn) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention) as well as `eager`, which is a simple matrix multiplication without any optimization on top. This is the setting you can usually choose when instantiating a model: