Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/source/en/attention_interface.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ supported models.

Most recent models can now switch from one attention function used in the Attention layer to the other, thanks to a simple mapping.
By default, we provide the implementation for [`sdpa`](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html),
[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention)
[`flash_attention_2`](https://github.com/Dao-AILab/flash-attention), [`flash_dynamic_mask_attention`](https://github.com/SmallDoges/flash-dmattn) and [`flex_attention`](https://pytorch.org/docs/stable/nn.attention.flex_attention.html#module-torch.nn.attention.flex_attention)
as well as `eager`, which is a simple matrix multiplication without any optimization on top.
This is the setting you can usually choose when instantiating a model:

Expand Down
83 changes: 83 additions & 0 deletions src/transformers/integrations/flash_dynamic_mask_attention.py

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same we will handle this then

Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
from typing import Optional

import torch

from ..modeling_flash_dynamic_mask_attention_utils import _flash_dynamic_mask_attention_forward
from ..utils import logging


logger = logging.get_logger(__name__)


def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtype:
"""If the query is in float32, return a target dtype compatible with flash dynamic mask attention. Return None otherwise."""
if query.dtype == torch.float32:
if torch.is_autocast_enabled():
return torch.get_autocast_gpu_dtype()
# Handle the case where the model is quantized
elif hasattr(module.config, "_pre_quantization_dtype"):
return module.config._pre_quantization_dtype
else:
return next(layer for layer in module.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype
return None


def flash_dynamic_mask_attention_forward(
module: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: Optional[torch.Tensor],
attention_bias: Optional[torch.Tensor],
scaling: Optional[float] = None,
softcap: Optional[float] = None,
is_causal: Optional[bool] = None,
**kwargs,
) -> tuple[torch.Tensor, None]:
if kwargs.get("output_attentions", False):
logger.warning_once(
"`flash_dynamic_mask_attention` does not support `output_attentions=True`."
" Please set your attention to `eager` if you want any of these features."
)

# This is before the transpose
seq_len = query.shape[2]

if any(dim == 0 for dim in query.shape):
raise ValueError(
"Tensor query has shape with a zero dimension.\n"
"FlashDynamicMaskAttention does not support inputs with dim=0.\n"
"Please check your input shapes or use SDPA instead."
)
# FDMA uses non-transposed inputs
query = query.transpose(1, 2)
key = key.transpose(1, 2)
value = value.transpose(1, 2)

# In PEFT, usually we cast the layer norms in float32 for training stability reasons
# therefore the input hidden states gets silently casted in float32. Hence, we need
# cast them back in the correct dtype just to be sure everything works as expected.
# This might slowdown training & inference so it is recommended to not cast the LayerNorms
# in fp32. (usually our RMSNorm modules handle it correctly)
target_dtype = get_target_dtype(query, module)

# Instead of relying on the value set in the module directly, we use the is_causal passed in kwargs if it is presented
is_causal = is_causal if is_causal is not None else module.is_causal

attn_output = _flash_dynamic_mask_attention_forward(
query,
key,
value,
attention_mask,
attention_bias,
query_length=seq_len,
is_causal=is_causal,
softmax_scale=scaling,
softcap=softcap,
target_dtype=target_dtype,
implementation=module.config._attn_implementation,
layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None,
**kwargs,
)

return attn_output, None
239 changes: 239 additions & 0 deletions src/transformers/modeling_flash_dynamic_mask_attention_utils.py

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here, we would handle most of this. Maybe small modifications would be needed to support the attention_bias parameter in https://github.com/huggingface/transformers/blob/main/src/transformers/modeling_flash_attention_utils.py

Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
# Copyright 2025 Jingze Shi and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import os
from functools import partial
from typing import Optional

import torch

from .utils import (
is_flash_dmattn_available,
logging,
)


logger = logging.get_logger(__name__)


# `globals()` is not compatible with dynamo, hence we have do define them in global scope ourselves
_fdma_fn = None

# function that processes kwargs, generalized to handle any supported kwarg within the function
_process_flash_kwargs_fn = None
# exceptions where hf API doesn't match the original FDMA API
_hf_api_to_flash_mapping = {
"dropout": None,
"sliding_window": None,
}


def _lazy_imports(implementation: Optional[str]):
"""
Lazy loads the respective flash dynamic mask attention implementations.

Return:
flash_dmattn_func: The base flash dynamic mask attention function.
"""
is_fdma = is_flash_dmattn_available()

if (implementation == "flash_dynamic_mask_attention" and is_fdma) or (implementation is None and is_fdma):
from flash_dmattn import flash_dmattn_func

return flash_dmattn_func


def _lazy_define_process_function(flash_function):
"""
Depending on the version and kernel some features are not supported. Due to limitations in
`torch.compile`, we opt to statically type which (optional) kwarg parameters are supported
within `_process_flash_dynamic_mask_attention_kwargs`.

NOTE: While all supported kwargs are marked as `True`, everything else is marked as `False`.
This might be confusing for kwargs that we use in any case, e.g. `is_causal`.
"""

flash_parameters = inspect.signature(flash_function).parameters
process_parameters = inspect.signature(_process_flash_dynamic_mask_attention_kwargs).parameters

supports_mapping = {}
for param in process_parameters:
fdma_param = _hf_api_to_flash_mapping.get(param, param)
supports_mapping[fdma_param] = fdma_param in flash_parameters

return partial(_process_flash_dynamic_mask_attention_kwargs, supports_mapping=supports_mapping)


def lazy_import_flash_dynamic_mask_attention(implementation: Optional[str], force_import: Optional[bool] = False):
"""
Lazily import flash dmattn and return the respective functions + flags.

NOTE: For fullgraph, this needs to be called before compile, while no fullgraph can
work without preloading. See `load_and_register_kernel` in `integrations.hub_kernels`.
"""
global _fdma_fn
if force_import or any(k is None for k in [_fdma_fn]):
_fdma_fn = _lazy_imports(implementation)

global _process_flash_kwargs_fn
if force_import or _process_flash_kwargs_fn is None:
_process_flash_kwargs_fn = _lazy_define_process_function(_fdma_fn)

return (_fdma_fn), _process_flash_kwargs_fn


def fdma_peft_integration_check(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
bias: Optional[torch.Tensor],
target_dtype: Optional[torch.dtype] = None,
):
"""
PEFT usually casts the layer norms in float32 for training stability reasons
therefore the input hidden states gets silently casted in float32. Hence, we need
cast them back in float16 / bfloat16 just to be sure everything works as expected.
This might slowdown training & inference so it is recommended to not cast the LayerNorms!
"""
if target_dtype and q.dtype == torch.float32:
logger.warning_once(f"Casting fp32 inputs back to {target_dtype} for flash-dmattn compatibility.")
q, k, v = q.to(target_dtype), k.to(target_dtype), v.to(target_dtype)
if bias is not None:
bias = bias.to(target_dtype)
return q, k, v, bias


def _process_flash_dynamic_mask_attention_kwargs(
query_length: int,
key_length: int,
is_causal: bool,
softmax_scale: Optional[float] = None,
softcap: Optional[float] = None,
deterministic: Optional[bool] = None,
s_aux: Optional[torch.Tensor] = None,
supports_mapping: Optional[dict[str, bool]] = None,
**kwargs,
):
"""
Returns a set of kwargs that are passed down to the according flash dynamic mask attention function based on
requested features and whether it is supported - depends on the version and kernel implementation
which is dynamically configured at `lazy_import_flash_dynamic_mask_attention`. The (un)supported features can be
inspected in `supports_mapping`, see `_lazy_define_process_function` for more details.

Args:
query_length (`int`):
Length of the query states
key_length (`int`):
Length of the key states
is_causal (`bool`):
Whether we perform causal (decoder) attention or full attention.
softmax_scale (`float`, *optional*):
The scaling of QK^T before applying softmax. Default to `1 / sqrt(head_dim)`.
softcap (`float`, *optional*):
Softcap for the attention logits, used e.g. in gemma2.
deterministic (`bool`, *optional*):
Determines if the deterministic option introduced in flash_dmattn>=1.0.0 is enabled.
s_aux (`torch.Tensor`, *optional*):
Attention sink auxiliary that adds a `bias` to the attention calculation via an additional head.
Return:
flash_kwargs (`dict`):
A dict of kwargs that are requested and supported.
"""
flash_kwargs = {
"is_causal": is_causal and query_length != 1,
"softmax_scale": softmax_scale,
}

if supports_mapping["deterministic"]:
flash_kwargs["deterministic"] = (
deterministic if deterministic is not None else os.getenv("FLASH_DMATTN_DETERMINISTIC", "0") == "1"
)

if supports_mapping["softcap"] and softcap is not None:
flash_kwargs["softcap"] = softcap

# Only within kernel implementation atm
if supports_mapping["s_aux"] and s_aux is not None:
flash_kwargs["s_aux"] = s_aux

return flash_kwargs


def _flash_dynamic_mask_attention_forward(
query_states: torch.Tensor,
key_states: torch.Tensor,
value_states: torch.Tensor,
attention_mask: Optional[torch.Tensor],
attention_bias: Optional[torch.Tensor],
query_length: int,
is_causal: bool,
softmax_scale: Optional[float] = None,
softcap: Optional[float] = None,
deterministic: Optional[bool] = None,
target_dtype: Optional[torch.dtype] = None,
implementation: Optional[str] = None,
**kwargs,
):
"""
Calls the forward method of Flash Dynamic Mask Attention - if the input hidden states contain at least one padding token
first unpad the input, then computes the attention scores and pad the final attention scores.

(Optional) kwargs are described further in `_process_flash_dynamic_mask_attention_kwargs` and `FlashDynamicMaskAttentionKwargs`.

Args:
query_states (`torch.Tensor`):
Input query states to be passed to Flash Dynamic Mask Attention API
key_states (`torch.Tensor`):
Input key states to be passed to Flash Dynamic Mask Attention API
value_states (`torch.Tensor`):
Input value states to be passed to Flash Dynamic Mask Attention API
attention_mask (`torch.Tensor`, *optional*):
The padding mask - corresponds to a tensor of size `(batch_size, num_heads, query_len, key_len)` where 0 stands for the
position of padding tokens and 1 for the position of non-padding tokens.
attention_bias (`torch.Tensor`, *optional*):
The attention bias tensor of size `(batch_size, num_heads, query_len, key_len)` to add to attention scores.
Comment on lines +204 to +205

Copilot AI Oct 27, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The attention_bias parameter is documented in the docstring but is not listed in the function signature's Args section. This parameter should be included with its description in the Args list for consistency with other parameters like attention_mask.

Copilot uses AI. Check for mistakes.
implementation (`str`, *optional*):
The attention implementation to use. If None, will default to the one based on the environment.
"""

(fdma_fn), process_flash_kwargs_fn = lazy_import_flash_dynamic_mask_attention(implementation)

# PEFT possibly silently casts tensors to fp32, this potentially reconverts to correct dtype or is a no op
query_states, key_states, value_states, attention_bias = fdma_peft_integration_check(
query_states, key_states, value_states, attention_bias, target_dtype
)

# Extract the flash dynamic mask attention kwargs that have been requested (and are supported by the implementation)
flash_kwargs = process_flash_kwargs_fn(
query_length=query_length,
key_length=key_states.size(1),
is_causal=is_causal,
softmax_scale=softmax_scale,
softcap=softcap,
deterministic=deterministic,
**kwargs,
)

out = fdma_fn(
query_states,
key_states,
value_states,
attention_mask,
attention_bias,
**flash_kwargs,
)
if isinstance(out, tuple):
out = out[0]

return out
Loading