Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions atom/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1978,6 +1978,11 @@ def qrep_for_step(
return qrep_enabled and not use_seg_mla and (not is_prefill or prefill_qrep)


# The FP4 cache dtypes: "fp4" is MXFP4 (e2m1 + e8m0 per 32), "nvfp4" is NVFP4
# (e2m1 + e4m3 per 16).
FP4_CACHE_DTYPES = ("fp4", "nvfp4")


def _check_fp4_kv(kv_cache_dtype: str, *, is_m3: bool, is_v41: bool) -> None:
"""``--kv-cache-dtype fp4`` is MiniMax-M3.1's to ask for and DeepSeek-V4.1's own;
refused for every other model. The guard admits the MiniMax-M3 architecture and
Expand All @@ -1987,13 +1992,15 @@ def _check_fp4_kv(kv_cache_dtype: str, *, is_m3: bool, is_v41: bool) -> None:
raise ValueError(
"--kv-cache-dtype fp4 is supported by MiniMax-M3.1 and DeepSeek-V4.1 only"
)
if kv_cache_dtype == "nvfp4" and not is_m3:
raise ValueError("--kv-cache-dtype nvfp4 is supported by MiniMax-M3 only")


def resolve_draft_kv_dtype(target_dtype: str, *, own_pool: bool) -> str:
"""The KV dtype of a speculative draft's pages: the target's, except bf16 for a
draft with a pool of its own (``own_pool``) under an fp4 target, which has no
draft with a pool of its own (``own_pool``) under an FP4 target, which has no
draft layout. A draft in the target's pool has the target's dtype."""
return "bf16" if own_pool and target_dtype == "fp4" else target_dtype
return "bf16" if own_pool and target_dtype in FP4_CACHE_DTYPES else target_dtype


def indexer_cp_unsupported_reason(
Expand All @@ -2004,6 +2011,7 @@ def indexer_cp_unsupported_reason(
dcp_size: int,
enable_tbo: bool,
plugin_mode: bool = False,
nvfp4_caches: bool = False,
) -> str | None:
"""Why MiniMax-M3 indexer-only context parallelism cannot run here, or None.

Expand Down Expand Up @@ -2031,6 +2039,8 @@ def indexer_cp_unsupported_reason(
# error. ``indexer_cp_enabled`` refuses them at runtime; this clears the
# flag so nothing downstream (log line, compile hash) disagrees.
return "plugin mode (the vLLM/SGLang bridges keep the TP indexer path)"
if nvfp4_caches:
return "the NVFP4 caches are not wired to the CP scorer"
if dcp_size > 1:
return (
"decode_context_parallel_size > 1 (KV-cache DCP owns the context "
Expand Down Expand Up @@ -3043,6 +3053,7 @@ def __post_init__(self):
self.decode_context_parallel_size,
self.enable_tbo or self.enable_tbo_decode,
is_plugin_mode(),
"nvfp4" in (self.index_cache_dtype, self.kv_cache_dtype),
)
if indexer_cp_off is not None:
logger.warning(
Expand Down Expand Up @@ -3084,11 +3095,15 @@ def __post_init__(self):
# load. Overriding it here would accept the flag and ignore it.
self.index_cache_dtype = "fp8"
else:
# M3's fp4 KV needs the fp4 index (the MXFP4 attention reads the
# M3's FP4 KV needs the matching FP4 index (the FP4 attention reads the
# selector's rows, which only that scorer sizes to match), which is
# what following the KV dtype gives; an explicit other index dtype
# is refused by `validate_runtime_config`.
self.index_cache_dtype = self.kv_cache_dtype
if self.index_cache_dtype == "nvfp4" and not is_m3:
raise ValueError(
"--index-cache-dtype nvfp4 is supported by MiniMax-M3 only"
)
if self.speculative_config is not None:
from atom.spec_decode.draft_kv import draft_owns_kv_pool

Expand Down
12 changes: 7 additions & 5 deletions atom/model_engine/arg_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,23 +320,25 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
)
parser.add_argument(
"--kv_cache_dtype",
choices=["bf16", "fp8", "fp4"],
choices=["bf16", "fp8", "fp4", "nvfp4"],
type=str,
default="bf16",
help="KV cache type. Default is 'bf16'. 'fp4' is MiniMax-M3.1's MXFP4 K/V "
"or DeepSeek-V4.1's own FP4 KV; other models refuse it. M3's conditions "
"are in the configuration guide.",
"or DeepSeek-V4.1's own FP4 KV; 'nvfp4' is MiniMax-M3.1's NVFP4 K/V "
"(needs --index-cache-dtype nvfp4); other models refuse them. M3's "
"conditions are in the configuration guide.",
)
parser.add_argument(
"--index-cache-dtype",
"--index_cache_dtype",
choices=["bf16", "fp8", "fp4"],
choices=["bf16", "fp8", "fp4", "nvfp4"],
type=str,
default=None,
help="Index cache type. Native single-node DeepSeek-V4 defaults to "
"'fp4' except on gfx942, which defaults to 'fp8'; other models "
"default to --kv_cache_dtype. 'fp4' is MiniMax-M3's MXFP4 index "
"cache or another model's own FP4 layout; see the configuration guide.",
"cache or another model's own FP4 layout; 'nvfp4' is MiniMax-M3's NVFP4 index "
"cache (e2m1 + e4m3 per 16); see the configuration guide.",
)
parser.add_argument(
"--block-size", type=int, default=16, help="KV cache block size."
Expand Down
99 changes: 79 additions & 20 deletions atom/model_ops/attention_mha.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from aiter.ops.triton.unified_attention import unified_attention
from torch import nn

from atom.config import get_current_atom_config
from atom.config import FP4_CACHE_DTYPES, get_current_atom_config
from atom.model_ops.base_attention import (
PA_ASM_MAX_QUERY_GROUP_SIZE,
cp_mha_gather_cache,
Expand Down Expand Up @@ -1218,7 +1218,7 @@ def forward(

# What aiter's fused QK-norm / rope / cache writer calls a cache format (anything
# else it takes as "auto"): ATOM's `fp4` is its "mxfp4". One table for both caches.
_AITER_WRITER_DTYPE = {"fp8": "fp8", "fp4": "mxfp4"}
_AITER_WRITER_DTYPE = {"fp8": "fp8", "fp4": "mxfp4", "nvfp4": "nvfp4"}


class SparseMHAPagedAttentionImpl(PagedAttentionImpl):
Expand Down Expand Up @@ -1360,6 +1360,10 @@ def __init__(
# here, not per call: it needs FlyDSL, which other caches / families lack
self._sparse_pa_mxfp4 = sparse_pa_mxfp4
self._select_page_kwargs = {"pages_per_block": 1, "block_page_stride": 1}
elif self.kv_cache_dtype == "nvfp4":
from atom.model_ops.minimax_m3.nvfp4_pages import Nvfp4KvLayout

self.kv_layout = Nvfp4KvLayout()
# Bound by AiterAttentionMetadataBuilder.build_kv_cache_tensor (Task 6):
# the page-128 indexer-key cache. None until the runner binds it.
self.index_cache: torch.Tensor | None = None
Expand Down Expand Up @@ -1405,7 +1409,8 @@ def rope_cache(self, q, k, v, qkv, position, fwd_ctx: ForwardContext):
from atom.model_ops.minimax_m3.sparse_attn import to_page16_shuffle

layer = kv_cache_data[f"layer_{self.layer_num}"]
if self.kv_cache_dtype == "fp4":
if self.kv_cache_dtype in ("fp4", "nvfp4"):
# nvfp4: one page buffer [K side | V side], k_cache and v_cache are the same
k_cache, v_cache, k_scale, v_scale = (
layer.k_cache,
layer.v_cache,
Expand Down Expand Up @@ -1437,24 +1442,29 @@ def rope_cache(self, q, k, v, qkv, position, fwd_ctx: ForwardContext):
# per-token dequant scales into k_scale / v_scale (outputs).
fused_k_scale = k_scale if is_fp8 else None
fused_v_scale = v_scale if is_fp8 else None
if self.kv_cache_dtype == "nvfp4":
# the layer's fp32 [1] global K / V scales, which the reader takes as well
fused_k_scale, fused_v_scale = self.nvfp4_k_scale, self.nvfp4_v_scale
fused_index_cache_dtype = (
"auto" # no index branch
if self.skip_index_topk
else _AITER_WRITER_DTYPE.get(self.index_cache_dtype, "auto")
)

if self.kv_cache_dtype == "fp4":
if self.kv_cache_dtype in FP4_CACHE_DTYPES:
fused_kv_block_size = SPARSE_BLOCK_SIZE
else:
# SHUFFLE page size (== ASM_PAGE_SIZE == 16)
fused_kv_block_size = k_cache.shape[3]
# e4m3 (cast as is) for the MXFP4 index's fp8-q scorer; the top-k reuse key
# e4m3 (cast as is) for an FP4 index's fp8-q scorer; the top-k reuse key
# records it, so a reusing layer finds what a producing layer stored
index_q_dtype = (
torch.float8_e4m3fn if self.index_cache_dtype == "fp4" else qkv.dtype
torch.float8_e4m3fn
if self.index_cache_dtype in FP4_CACHE_DTYPES
else qkv.dtype
)

if self.skip_index_topk and self.kv_cache_dtype != "fp4":
if self.skip_index_topk and self.kv_cache_dtype not in FP4_CACHE_DTYPES:
from atom.model_ops.triton_fused_qkv_norm_rope_cache import (
triton_fused_norm_rope_cache,
)
Expand Down Expand Up @@ -1529,7 +1539,7 @@ def rope_cache(self, q, k, v, qkv, position, fwd_ctx: ForwardContext):
index_cache_dtype=fused_index_cache_dtype,
k_scale=fused_k_scale,
v_scale=fused_v_scale,
asm_layout=self.kv_cache_dtype != "fp4",
asm_layout=self.kv_cache_dtype not in FP4_CACHE_DTYPES,
skip_index_branch=self.skip_index_topk,
)

Expand Down Expand Up @@ -1655,23 +1665,23 @@ def _store_cached_topk(self, sparse_metadata, key: tuple, value: tuple):
}

def _index_scorer(self, sparse_metadata):
"""The step's MXFP4 index scorer record, None for any other index cache; a layer
of the MXFP4 index whose metadata has none raises (Triton would read uint8 pages).
"""The step's FP4 index scorer record, None for any other index cache; a layer
of an FP4 index whose metadata has none raises (Triton would read uint8 pages).
"""
if self.index_cache_dtype != "fp4":
if self.index_cache_dtype not in FP4_CACHE_DTYPES:
return None
if sparse_metadata.index_scorer is None:
raise ValueError(
"the MXFP4 index cache needs the step's index_scorer on the sparse "
"the FP4 index cache needs the step's index_scorer on the sparse "
"metadata"
)
return sparse_metadata.index_scorer

def _metadata_rows(self, q, covered: int) -> int:
"""The rows of ``q`` this step's sparse metadata covers: what the selector writes
and so all attention may read, in decode and prefill (a DP step can hand ``q``
more). Only the MXFP4 index keeps to it; the others run every row."""
if self.index_cache_dtype != "fp4":
more). Only the FP4 indexes keep to it; the others run every row."""
if self.index_cache_dtype not in FP4_CACHE_DTYPES:
return q.shape[0]
assert covered <= q.shape[0], (covered, q.shape)
return covered
Expand Down Expand Up @@ -1717,7 +1727,8 @@ def _sparse_prefill(
# exactly the head this rank would have projected alone, so the
# slice is an identity, not an approximation.
index_q = index_q[:, self.indexer_cp_rank : self.indexer_cp_rank + 1]
topk_idx, sparse_bt, sparse_ctx = minimax_m3_index_topk(
nvfp4 = self.kv_cache_dtype == "nvfp4"
selected = minimax_m3_index_topk(
index_q,
self.index_cache,
block_tables,
Expand All @@ -1731,11 +1742,15 @@ def _sparse_prefill(
self.local_blocks,
self.num_kv_heads,
self.scale,
emit_sparse_block_table=True,
# the NVFP4 attention reads the logical block ids, no page-16 table
emit_sparse_block_table=not nvfp4,
n_valid_column_per_row=sparse_metadata.n_valid_column_per_row,
index_scorer=self._index_scorer(sparse_metadata),
**self._select_page_kwargs,
)
topk_idx, sparse_bt, sparse_ctx = (
(selected, None, None) if nvfp4 else selected
)
self._store_cached_topk(
sparse_metadata, topk_key, (topk_idx, sparse_bt, sparse_ctx)
)
Expand All @@ -1746,7 +1761,30 @@ def _sparse_prefill(
output = torch.empty_like(q, dtype=v.dtype if fp4_kv else None)
if rows < q.shape[0]:
output[rows:].zero_()
if fp4_kv:
if self.kv_cache_dtype == "nvfp4":
from atom.model_ops.minimax_m3.sparse_attn_nvfp4 import (
prefill_row_positions,
sparse_pa_nvfp4_prefill,
)

# every layer of the step has the same rows: built once, on the metadata
row_info = getattr(prefill_md, "nvfp4_row_info", None)
if row_info is None:
row_info = prefill_md.nvfp4_row_info = prefill_row_positions(
cu_seqlens_q, prefix_lens, rows
)
sparse_pa_nvfp4_prefill(
q[:rows],
k_cache,
topk_idx,
block_tables,
*row_info,
self.scale,
self.nvfp4_k_scale,
self.nvfp4_v_scale,
out=output[:rows],
)
elif fp4_kv:
# plan-free: a prefill chunk's row count is new almost every forward
self._sparse_pa_mxfp4(
q[:rows],
Expand Down Expand Up @@ -1913,7 +1951,8 @@ def _sparse_decode(
index_scorer=self._index_scorer(sparse_metadata),
)
else:
topk_idx, sparse_bt, sparse_ctx = minimax_m3_index_topk_decode(
nvfp4 = self.kv_cache_dtype == "nvfp4"
selected = minimax_m3_index_topk_decode(
index_q,
self.index_cache,
decode_md.block_table,
Expand All @@ -1924,12 +1963,15 @@ def _sparse_decode(
self.local_blocks,
self.num_kv_heads,
self.scale,
emit_sparse_block_table=True,
emit_sparse_block_table=not nvfp4,
max_query_len=max_query_len,
n_valid_column_per_row=sparse_metadata.n_valid_column_per_row,
index_scorer=self._index_scorer(sparse_metadata),
**self._select_page_kwargs,
)
topk_idx, sparse_bt, sparse_ctx = (
(selected, None, None) if nvfp4 else selected
)
self._store_cached_topk(
sparse_metadata, topk_key, (topk_idx, sparse_bt, sparse_ctx)
)
Expand All @@ -1941,7 +1983,24 @@ def _sparse_decode(
output = torch.empty_like(q, dtype=v.dtype if fp4_kv else None)
if rows < q.shape[0]:
output[rows:].zero_()
if fp4_kv:
if self.kv_cache_dtype == "nvfp4":
from atom.model_ops.minimax_m3.sparse_attn_nvfp4 import (
sparse_pa_nvfp4_decode,
)

sparse_pa_nvfp4_decode(
q[:rows],
k_cache,
topk_idx,
decode_md.block_table,
decode_md.seq_lens,
self.scale,
self.nvfp4_k_scale,
self.nvfp4_v_scale,
max_query_len,
output[:rows],
)
elif fp4_kv:
self._sparse_pa_mxfp4(
q[:rows],
k_cache,
Expand Down
28 changes: 27 additions & 1 deletion atom/model_ops/attentions/aiter_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,9 +508,24 @@ def _index_cache_spec(self, entry_tokens: int) -> _IndexCacheSpec | None:
# the one geometry check of an MXFP4 page, as the K / V layout makes it
Mxfp4KvLayout().check(self.block_size, dim, entry_tokens // self.block_size)
return _IndexCacheSpec(rows, (PAGE_BYTES,), torch.uint8, (-1, PAGE_BYTES))
if index_dtype == "nvfp4":
from atom.model_ops.minimax_m3.nvfp4_pages import (
HEAD_DIM,
INDEX_PAGE_BYTES,
)

if dim != HEAD_DIM or entry_tokens != self.block_size:
raise ValueError(
"the NVFP4 index cache is 128-wide keys, one block an entry; "
f"got dim {dim}, {entry_tokens} tokens an entry"
)
return _IndexCacheSpec(
rows, (INDEX_PAGE_BYTES,), torch.uint8, (-1, INDEX_PAGE_BYTES)
)
if index_dtype not in ("bf16", "fp8"):
raise ValueError(
f"index_cache_dtype must be 'bf16', 'fp8' or 'fp4', got {index_dtype!r}"
"index_cache_dtype must be 'bf16', 'fp8', 'fp4' or 'nvfp4', got "
f"{index_dtype!r}"
)
dtype = dtypes.d_dtypes[index_dtype]
return _IndexCacheSpec(
Expand Down Expand Up @@ -629,6 +644,17 @@ def build_kv_cache_tensor(self, module):
if pool.scale_fields:
module.k_scale, module.v_scale = pool.scale_views(row)

if config.kv_cache_dtype == "nvfp4" and _is_indexed_sparse_attention(module):
# The layer's fp32 global K / V scale, one element each, outside the
# page. 1.0: the M3.1 checkpoint carries no k_scale / v_scale and vLLM's
# default is the same; the kernels take whatever value is here.
module.impl.nvfp4_k_scale = torch.ones(
1, dtype=torch.float32, device=k_cache.device
)
module.impl.nvfp4_v_scale = torch.ones(
1, dtype=torch.float32, device=k_cache.device
)

module.max_model_len = config.max_model_len
module.k_cache = k_cache
module.v_cache = v_cache
Expand Down
Loading
Loading