Skip to content
Open
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
d8e4c4f
sdpa x varlen fa
vasqu Mar 31, 2026
5c057ff
multiple fixes
vasqu Mar 31, 2026
a763e02
disable generate on non mask path
vasqu Mar 31, 2026
ca2ae0c
circumvent mask creation on packed sequences if varlen available
vasqu Mar 31, 2026
4470fe8
fix
vasqu Mar 31, 2026
2aac274
qwen example
vasqu Mar 31, 2026
dc866c8
fix this
vasqu Mar 31, 2026
ef5d6a6
style, docs, and remove mask skip on sdpa
vasqu Mar 31, 2026
93c239c
wtf
vasqu Mar 31, 2026
18af76f
knew it
vasqu Mar 31, 2026
6c09baf
might revert, test if the varlen fallback is aight
vasqu Mar 31, 2026
a28a9bf
fix
vasqu Mar 31, 2026
cabd09b
deprecate top left mask (was already done somewhat before as its unus…
vasqu Apr 1, 2026
987c269
oops
vasqu Apr 1, 2026
5fb4582
make alt names list of str, small nit
vasqu Apr 1, 2026
7a0ce68
docstrings for support mapping
vasqu Apr 1, 2026
f43f855
fix these
vasqu Apr 1, 2026
25bc50e
important fix
vasqu Apr 1, 2026
0e4f5ee
revert qwen vl for now
vasqu Apr 1, 2026
aaff872
forgot about this one
vasqu Apr 1, 2026
6ea0a0e
Merge branch 'main' into torch-flash-attn
vasqu Apr 1, 2026
91cde1c
deprecate stuff properly
vasqu Apr 1, 2026
7ac6072
fix
vasqu Apr 1, 2026
ffb0f2a
Merge branch 'main' into torch-flash-attn
vasqu Jul 28, 2026
05aae16
fix deprecation cycle + add proper varlen path
vasqu Jul 28, 2026
54e2bd0
style
vasqu Jul 28, 2026
151e7f3
reshape at the end
vasqu Jul 28, 2026
10e34c7
tests
vasqu Jul 28, 2026
202e873
remove support for these models
vasqu Jul 28, 2026
3a5b41c
proper fixups on compatible fa implementations
vasqu Jul 28, 2026
5d06085
revert
vasqu Jul 28, 2026
6aa8493
remove sdpa/fa interleave option
vasqu Jul 29, 2026
0e65665
Merge branch 'main' into torch-flash-attn
vasqu Jul 29, 2026
c6df671
style
vasqu Jul 29, 2026
8b77f46
best effort fix
vasqu Jul 29, 2026
8469405
add amd kernel alternative
vasqu Jul 30, 2026
60e6316
Merge branch 'main' into torch-flash-attn
vasqu Jul 30, 2026
186558f
Merge branch 'main' into torch-flash-attn
vasqu Sep 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,9 @@ def pytest_configure(config):
config.addinivalue_line("markers", "flash_attn_test: mark test which tests flash attention functionality")
config.addinivalue_line("markers", "flash_attn_3_test: mark test which tests flash attention 3 functionality")
config.addinivalue_line("markers", "flash_attn_4_test: mark test which tests flash attention 4 functionality")
config.addinivalue_line(
"markers", "flash_attn_torch_test: mark test which tests flash attention torch functionality"
)
config.addinivalue_line(
"markers", "all_flash_attn_test: mark test which tests all mainline flash attentions' functionality"
)
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ addopts = "--doctest-glob='**/*.md'"
doctest_optionflags="NUMBER NORMALIZE_WHITESPACE ELLIPSIS"
markers = [
"all_flash_attn_test: marks tests related to all mainline flash attentions at once (deselect with '-m \"not all_flash_attn_test\"')",
"flash_attn_torch_test: marks tests related to flash attention torch (deselect with '-m \"not flash_attn_torch_test\"')",
"flash_attn_4_test: marks tests related to flash attention 4 (deselect with '-m \"not flash_attn_4_test\"')",
"flash_attn_3_test: marks tests related to flash attention 3 (deselect with '-m \"not flash_attn_3_test\"')",
"flash_attn_test: marks tests related to flash attention (deselect with '-m \"not flash_attn_test\"')",
Expand Down
5 changes: 1 addition & 4 deletions src/transformers/integrations/flash_attention.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
import torch

from ..modeling_flash_attention_utils import _flash_attention_forward, flash_attn_supports_top_left_mask
from ..modeling_flash_attention_utils import _flash_attention_forward
from ..utils import logging


logger = logging.get_logger(__name__)

_use_top_left_mask = flash_attn_supports_top_left_mask()

Comment on lines -9 to -10

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it not needed anymore? We went beyond the versions that required this hack?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is already silently not used anymore except for NPU users that set an environment variable. Imo, we should deprecate this (maybe a different PR but I need some cleanups here either way)

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bump ^


def get_target_dtype(query: torch.Tensor, module: torch.nn.Module) -> torch.dtype:
"""If the query is in float32, return a target dtype compatible with flash attention. Return None otherwise."""
Expand Down Expand Up @@ -84,7 +82,6 @@ def flash_attention_forward(
softmax_scale=scaling,
sliding_window=sliding_window,
softcap=softcap,
use_top_left_mask=_use_top_left_mask,
target_dtype=target_dtype,
attn_implementation=module.config._attn_implementation,
layer_idx=module.layer_idx if hasattr(module, "layer_idx") else None,

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was not used for quite a while now and just hidden behind kwargs

Expand Down
17 changes: 11 additions & 6 deletions src/transformers/integrations/npu_flash_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

import math
import os
import warnings

import torch

Expand All @@ -23,15 +24,14 @@


# FlashAttention2 is supported on Ascend NPU with down-right aligned causal mask by default.
# Set environment variable `NPU_FA2_SPARSE_MODE` to 2 when using top-left aligned causal mask.
TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2
# It can set an environment variable `NPU_FA2_SPARSE_MODE` to control this behavior.
TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE = 2 # Deprecated

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecating top left mask: The min FA version of 2.3.3 no longer uses that and we should just align

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bump ^

DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE = 3

SPARSE_MODE = int(os.getenv("NPU_FA2_SPARSE_MODE", default=DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE))
if SPARSE_MODE not in [TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE, DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE]:
if SPARSE_MODE != DOWN_RIGHT_ALIGNED_CAUSAL_MASK_MODE:
raise ValueError(
"Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 2 (top-left aligned causal mask) "
"or 3 (down-right aligned causal mask)."
"Environment variable `NPU_FA2_SPARSE_MODE` can only be set as 3 (down-right aligned causal mask)."
)

ATTN_MASK_NPU_CACHE = {}
Expand All @@ -45,7 +45,12 @@ def get_attn_mask_npu(device):


def is_npu_fa2_top_left_aligned_causal_mask():
return SPARSE_MODE == TOP_LEFT_ALIGNED_CAUSAL_MASK_MODE if is_torch_npu_available() else False
warnings.warn(
"`is_npu_fa2_top_left_aligned_causal_mask` is deprecated and will be removed in v5.16. "
"This is no longer needed as the minimum required FA version is no longer affected by this.",
FutureWarning,
)
return False


def npu_flash_attn_func(
Expand Down
1 change: 1 addition & 0 deletions src/transformers/masking_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,7 @@ class AttentionMaskInterface(GeneralInterface):
"flash_attention_2": flash_attention_mask,
"flash_attention_3": flash_attention_mask,
"flash_attention_4": flash_attention_mask,
"flash_attention_torch": flash_attention_mask,
"flex_attention": flex_attention_mask,
}

Expand Down
Loading
Loading