From 2f682913287cfea38f9127d71ea14a63d512fe5d Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Wed, 4 Nov 2026 12:43:11 +0100 Subject: [PATCH 01/14] [Mamba] Add FlashInfer GDN ReplaySSM STP and MTP integration Build on the shared ReplaySSM lifecycle in #55688. Support Qwen3.5 BF16 replay state, fixed-width speculative verification, and aligned prefix materialization through existing FlashInfer APIs. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/speculative_decoding/mtp.md | 27 ++ tests/kernels/mamba/test_gdn_replayssm.py | 194 +++++++++++++ tests/kernels/mamba/test_ssu_dispatch.py | 3 + tests/model_executor/test_replayssm_warmup.py | 1 + tests/test_envs.py | 2 +- tests/v1/worker/test_gpu_batch_ordering.py | 24 +- .../worker/test_mamba_hybrid_model_state.py | 9 +- vllm/config/cache.py | 12 +- vllm/config/vllm.py | 141 +++++++++- vllm/envs.py | 11 +- vllm/model_executor/layers/mamba/gdn/base.py | 42 ++- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 120 +++++++- .../layers/mamba/gdn/replayssm.py | 266 ++++++++++++++++++ .../layers/mamba/mamba_utils.py | 43 +++ .../layers/mamba/ops/ssu_dispatch.py | 14 + vllm/model_executor/models/interfaces.py | 9 +- vllm/model_executor/models/qwen3_5.py | 70 ++--- vllm/model_executor/warmup/kernel_warmup.py | 3 + .../model_executor/warmup/replayssm_warmup.py | 120 +++++++- vllm/models/qwen3_5/nvidia/model.py | 68 ++--- vllm/v1/attention/backend.py | 6 + vllm/v1/attention/backends/gdn_attn.py | 111 +++++++- vllm/v1/worker/gpu/model_runner.py | 6 + .../worker/gpu/model_states/mamba_hybrid.py | 44 ++- vllm/v1/worker/gpu_model_runner.py | 23 +- vllm/v1/worker/mamba_utils.py | 16 +- vllm/v1/worker/replayssm_utils.py | 179 +++++++++--- 27 files changed, 1388 insertions(+), 176 deletions(-) create mode 100644 tests/kernels/mamba/test_gdn_replayssm.py create mode 100644 vllm/model_executor/layers/mamba/gdn/replayssm.py diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index 34d8f9539b74..5a75d28ad31a 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -67,6 +67,33 @@ vllm serve XiaomiMiMo/MiMo-7B-Base \ --speculative-config '{"method":"mtp","num_speculative_tokens":1}' ``` +## Qwen3.5 GDN ReplaySSM + +GDN ReplaySSM stores recent recurrent inputs and reconstructs state from checkpoints. +This optional FlashInfer backend is disabled by default. Enable it for Qwen3.5 with: + +```bash +VLLM_USE_V2_MODEL_RUNNER=1 SGLANG_GDN_WY_STRIDED_QKV=1 \ +VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm \ +vllm serve Qwen/Qwen3.5-0.8B \ + --dtype bfloat16 --mamba-cache-dtype bfloat16 \ + --mamba-ssm-cache-dtype bfloat16 --use-replayssm \ + --replayssm-buffer-len 16 --no-enable-prefix-caching \ + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' +``` + +This backend supports one to seven speculative tokens. It requires BF16 state and +128-element key and value heads. Pipeline parallelism, microbatching, and KV +connectors are unsupported. Adaptive verification is unsupported because GDN +output indices require fixed verification lengths. +Set `enable_adaptive_verification` to `false` in the speculative configuration. +Prefix caching requires `--mamba-cache-mode align`. + +FlashInfer must provide `gated_delta_rule_stp_ucache_flush` and +`gated_delta_rule_mtp_ucache_flush`. Prefix caching also requires +`gdn_prefix_materialize`. Check these APIs before use; a version number alone does +not establish availability. This integration does not install missing kernels. + ## Notes - MTP only works for model families that support MTP in vLLM. diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py new file mode 100644 index 000000000000..6da884c488f5 --- /dev/null +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math + +import pytest +import torch +import torch.nn.functional as F + +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + pack_replayssm_rows, + run_gdn_replayssm, +) + + +def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): + values = torch.tensor([[1.0], [2.0], [3.0]]) + packed = pack_replayssm_rows( + values, torch.tensor([0, 1, 3]), 4, padding_value=-torch.inf + ) + torch.testing.assert_close( + packed.squeeze(-1), + torch.tensor( + [ + [1.0, -torch.inf, -torch.inf, -torch.inf], + [2.0, 3.0, -torch.inf, -torch.inf], + ] + ), + ) + + +def _reference_gdn_step( + state, + q, + k, + v, + a, + b, + A_log, + dt_bias, +): + """FP32 recurrence using FlashInfer's effective BF16 gate constants.""" + state = state.float().clone() + A_log = A_log.to(torch.bfloat16).float() + dt_bias = dt_bias.to(torch.bfloat16).float() + group_size = v.shape[1] // k.shape[1] + outputs = [] + for token in range(q.shape[0]): + k_hv = F.normalize(k[token].float(), dim=-1).repeat_interleave( + group_size, dim=0 + ) + q_hv = F.normalize(q[token].float(), dim=-1).repeat_interleave( + group_size, dim=0 + ) + q_hv *= 1.0 / math.sqrt(q.shape[-1]) + decay = -torch.exp(A_log) * F.softplus(a[token].float() + dt_bias) + beta = torch.sigmoid(b[token].float()) + state *= torch.exp(decay)[:, None, None] + prediction = torch.einsum("hvk,hk->hv", state, k_hv) + update = (v[token].float() - prediction) * beta[:, None] + state += update[:, :, None] * k_hv[:, None, :] + outputs.append(torch.einsum("hvk,hk->hv", state, q_hv)) + return torch.stack(outputs), state + + +def _reconstruct_replayssm_state( + checkpoint, + k_cache, + u_cache, + g_cache, + committed, + ring_start, +): + state = checkpoint.float().clone() + if committed == 0: + return state + rows = torch.tensor( + [(ring_start + offset) % k_cache.size(1) for offset in range(committed)], + dtype=torch.long, + device=checkpoint.device, + ) + k_logical = k_cache.index_select(1, rows).float().repeat_interleave(2, dim=0) + u_logical = u_cache.index_select(1, rows).float() + g_logical = g_cache.index_select(1, rows).float() + final_g = g_logical[:, -1] + weights = torch.exp(final_g[:, None] - g_logical) + return torch.exp(final_g)[:, None, None] * state + torch.einsum( + "hpv,hpk->hvk", weights[:, :, None] * u_logical, k_logical + ) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() < (9, 0), + reason="FlashInfer GDN ReplaySSM requires an SM90+ CUDA device", +) +@pytest.mark.parametrize("width,query_len,accepted", [(1, 1, 1), (4, 3, 2), (8, 6, 4)]) +def test_gdn_replayssm_output_and_accepted_state( + monkeypatch, width, query_len, accepted +): + """Check nonzero outputs and accepted state through replay, wrap, and flush.""" + monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") + module = ( + "gdn_decode_bf16_wy_ucache_stp" + if width == 1 + else "gdn_decode_bf16_wy_ucache_flush" + ) + pytest.importorskip(f"flashinfer.gdn_kernels.{module}") + device, dtype = torch.device("cuda"), torch.bfloat16 + generator = torch.Generator(device=device).manual_seed(20260909 + width) + ring_slots = 16 if width == 1 else 32 + + def random_tensor(*shape): + return torch.randn(*shape, generator=generator, device=device).to(dtype) + + # Use one page stride for every state, as MambaBase.bind_kv_cache does. + shapes = [ + (2, 128, 128), + (2, ring_slots, 128), + (1, ring_slots, 128), + (2, ring_slots), + ] + dtypes = [dtype, dtype, dtype, torch.float32] + sizes = [math.prod(shape) * dt.itemsize for shape, dt in zip(shapes, dtypes)] + pages = torch.zeros(2, sum(sizes) + 16, dtype=torch.uint8, device=device) + states = [] + offset = 0 + for shape, dt, size in zip(shapes, dtypes, sizes): + states.append(pages[:, offset : offset + size].view(dt).view(2, *shape)) + offset += size + checkpoint, u_cache, k_cache, g_cache = states + checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) + ring_start = torch.zeros(2, dtype=torch.int32, device=device) + committed = torch.zeros_like(ring_start) + A_log = torch.full((2,), -3.0, device=device) + dt_bias = random_tensor(2).float() * 0.2 + accepted_state = checkpoint[1].float().clone() + + # Eighteen steps cross the native STP flush and the MTP ring boundary. + for _ in range(18): + q, k = random_tensor(query_len, 1, 128), random_tensor(query_len, 1, 128) + v = random_tensor(query_len, 2, 128) * 0.1 + a, b = random_tensor(query_len, 2) * 0.2, random_tensor(query_len, 2) + output = run_gdn_replayssm( + mixed_qkv=torch.cat((q.flatten(1), k.flatten(1), v.flatten(1)), dim=-1), + a=a, + b=b, + A_log=A_log, + dt_bias=dt_bias, + checkpoint_state=checkpoint, + replayssm_cache=(u_cache, k_cache, g_cache), + ring_start=ring_start, + num_committed=committed, + query_start_loc=torch.tensor( + [0, query_len], dtype=torch.int32, device=device + ), + state_indices=torch.tensor([1], dtype=torch.int32, device=device), + output_indices=torch.arange(query_len, dtype=torch.int32, device=device), + executed_query_width=width, + num_k_heads=1, + num_v_heads=2, + head_k_dim=128, + head_v_dim=128, + ) + expected_output, _ = _reference_gdn_step( + accepted_state, q, k, v, a, b, A_log, dt_bias + ) + torch.testing.assert_close(output.float(), expected_output, atol=8e-3, rtol=0) + _, accepted_state = _reference_gdn_step( + accepted_state, + q[:accepted], + k[:accepted], + v[:accepted], + a[:accepted], + b[:accepted], + A_log, + dt_bias, + ) + old_committed = int(committed[1].item()) + if width == 1: + committed[1] = 0 if old_committed >= 15 else old_committed + 1 + elif old_committed + width > 16: + ring_start[1] = (ring_start[1] + old_committed) % ring_slots + committed[1] = accepted + else: + committed[1] = old_committed + accepted + reconstructed = _reconstruct_replayssm_state( + checkpoint[1], + k_cache[1], + u_cache[1], + g_cache[1], + int(committed[1].item()), + int(ring_start[1].item()), + ) + torch.testing.assert_close(reconstructed, accepted_state, atol=2e-2, rtol=0) diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index f00f927f8356..fff23106e3ea 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -373,10 +373,13 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, + EXECUTED_QUERY_WIDTH=0, + GDN_STP=False, ) _compact_replayssm_requests_kernel[(1,)]( context.plan_flush_count, context.active_request_indices, + context.num_active, 4, MAX_NUM_REQS=4, BLOCK_SIZE=4, diff --git a/tests/model_executor/test_replayssm_warmup.py b/tests/model_executor/test_replayssm_warmup.py index 71d9d6430844..d72bd38d38e5 100644 --- a/tests/model_executor/test_replayssm_warmup.py +++ b/tests/model_executor/test_replayssm_warmup.py @@ -47,6 +47,7 @@ def _autotune_runner( use_replayssm and backend == MambaBackendEnum.FLASHINFER ), use_v2_model_runner=use_v2_model_runner, + is_gdn_replayssm_enabled=lambda: False, ), uniform_decode_query_len=query_len, decode_query_len=query_len, diff --git a/tests/test_envs.py b/tests/test_envs.py index 4298078278a8..2f3d0dc2f689 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -300,7 +300,7 @@ def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False) assert env_func() == "cuda" - for value in ("cuda", "triton"): + for value in ("cuda", "triton", "flashinfer_replayssm"): monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) assert env_func() == value diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 498ff7a728ce..51db644ab33d 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -48,13 +48,18 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata def _make_runner( - req_states: dict[str, tuple[int, int]], decode_query_len: int = 8 + req_states: dict[str, tuple[int, int]], + decode_query_len: int = 8, + use_gdn_replayssm: bool = False, ) -> Any: """Build a runner stub from {req_id: (num_computed_tokens, prefill_len)}.""" prefill_lens = np.array([s[1] for s in req_states.values()], dtype=np.int32) num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32) runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = decode_query_len + runner.vllm_config = SimpleNamespace( + is_gdn_replayssm_enabled=lambda: use_gdn_replayssm + ) runner.adaptive_verification = None runner.pcp_manager = None runner.use_pp = False @@ -154,6 +159,22 @@ def test_one_new_prompt_token_over_context_runs_as_decode(): } +def test_gdn_intermediate_single_token_then_multi_token_prompt_chunk(): + # Token 65 must update canonical state before the final three-token prefill. + runner = _make_runner({"prompt": (0, 68)}, 1, use_gdn_replayssm=True) + for computed, scheduled in [(0, 64), (64, 1), (65, 3)]: + runner.req_states.num_computed_prefill_tokens[0] = computed + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"prompt": scheduled}, + total_num_scheduled_tokens=scheduled, + scheduled_spec_decode_tokens={}, + ) + state, uniform = runner.gather_batch_req_state(scheduler_output, False) + assert state.prefill_runs_as_decode_np.tolist() == [False] + assert not state.decode_graph_eligible + assert uniform is None + + def test_one_token_prompt_tail_with_prior_context_is_uniform_decode(): # Replaying the last prompt token over existing context (full prefix-cache # hit or P/D KV import) computes exactly like a decode. @@ -245,6 +266,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): """ runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = 8 + runner.vllm_config = SimpleNamespace(is_gdn_replayssm_enabled=lambda: False) runner.adaptive_verification = None runner.pcp_manager = None runner.use_pp = False diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index 2cff25cc1a6c..8b0fbbacec28 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -66,12 +66,14 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non num_tokens_after_padding=1, query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(), query_start_loc=torch.tensor([0, 1], dtype=torch.int32), - num_scheduled_tokens=torch.tensor([1], dtype=torch.int32), + num_scheduled_tokens=np.array([1], dtype=np.int32), max_query_len=None, seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), seq_lens=torch.tensor([1537], dtype=torch.int32), is_prefilling_np=torch.tensor([False]).numpy(), prefill_runs_as_decode_np=None, + num_computed_prefill_tokens_np=np.array([1536]), + prefill_len_np=np.array([1024]), dcp_local_seq_lens=None, positions=positions, prompt_lens=torch.tensor([1024], dtype=torch.int32), @@ -146,6 +148,8 @@ def test_padded_prompt_tail_builds_as_spec_decode( seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), seq_lens=torch.tensor(seq_lens, dtype=torch.int32), is_prefilling_np=np.array(is_prefilling), + num_computed_prefill_tokens_np=np.array([46, 128, 0]), + prefill_len_np=np.array([40, 129, 8]), prefill_runs_as_decode_np=np.array([False, True, False]), dcp_local_seq_lens=None, positions=torch.zeros(12, dtype=torch.int64), @@ -218,6 +222,8 @@ def _input_batch( max_query_len=None, seq_lens_cpu_upper_bound=seq_lens, is_prefilling_np=np.array(is_prefilling), + num_computed_prefill_tokens_np=np.where(is_prefilling, 0, 10), + prefill_len_np=np.where(is_prefilling, 16, 10), prefill_runs_as_decode_np=None, num_draft_tokens_per_req=None if num_draft_tokens_per_req is None @@ -509,6 +515,7 @@ def test_flashinfer_replayssm_prefix_uses_original_accepted_counts() -> None: state = object.__new__(MambaHybridModelState) state._align_mode = True state._use_flashinfer_replayssm = True + state._use_gdn_replayssm = False state.recoverssm = None state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32, device="cuda") state._mamba_state_idx_gpu = torch.zeros(4, dtype=torch.int32, device="cuda") diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 7c01108cc171..f0a88c40f024 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -319,12 +319,14 @@ def device_memory_utilization(self, value: float | None) -> None: EAGLE on the Mamba group, and a prefix match unit smaller than the Mamba block size.""" replayssm_buffer_len: int = Field(default=16, gt=0) - """ReplaySSM logical history length B for Mamba2. Triton uses B physical - rows and FlashInfer uses B+T, where T is the target verification length. - Kimi-K3 speculative decode does not use B. Default 16.""" + """ReplaySSM logical history length B. Mamba2 Triton uses B physical rows + and Mamba2 FlashInfer uses B+T, where T is the verification length. GDN + FlashInfer requires B=16 and uses a 16-row physical ring for native STP or + a 32-row ring for MTP. Kimi-K3 speculative decode does not use B. Default + 16.""" use_replayssm: bool = False - """Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip - the per-step full-state store, writing the checkpoint back only on flush. + """Use a supported ReplaySSM decode kernel: cache recent recurrent inputs + and skip the per-step full-state store, writing the checkpoint only on flush. Triton supports 'none' and 'align' on Model Runner V1. FlashInfer supports 'none' and 'align' on Model Runner V1 and V2. Mamba2 speculative decoding requires FlashInfer. With prefix caching enabled, Triton supports diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 1e8f36826f7b..da35882e6917 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -128,6 +128,18 @@ } ) +# SupportsReplaySSM advertises support for some ReplaySSM implementation. This +# allowlist selects architectures validated for the FlashInfer GDN integration: +# sharing a layer or inheriting the marker does not validate the model lifecycle. +_GDN_REPLAYSSM_ARCHITECTURES = frozenset( + { + "Qwen3_5ForCausalLM", + "Qwen3_5MoeForCausalLM", + "Qwen3_5ForConditionalGeneration", + "Qwen3_5MoeForConditionalGeneration", + } +) + @lru_cache def default_breakable_cudagraph_architectures() -> frozenset[str]: @@ -638,9 +650,9 @@ def max_in_flight_tokens(self) -> int: @property def use_flashinfer_replayssm(self) -> bool: - return ( - self.cache_config.use_replayssm - and self.mamba_config.backend == MambaBackendEnum.FLASHINFER + return self.cache_config.use_replayssm and ( + VllmConfig.is_gdn_replayssm_enabled(self) + or self.mamba_config.backend == MambaBackendEnum.FLASHINFER ) @property @@ -3934,10 +3946,124 @@ def validate_mamba_block_size(self) -> "VllmConfig": @model_validator(mode="after") def validate_mamba_cached_kernel(self) -> "VllmConfig": + # Call the implementation on the class so this validator remains usable + # by focused tests that pass a config-shaped object rather than a full + # pydantic model. + use_gdn_replayssm = VllmConfig.is_gdn_replayssm_enabled(self) + is_gdn_replayssm_architecture = ( + self.model_config is not None + and self.model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES + ) + if ( + envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" + and not self.cache_config.use_replayssm + ): + raise ValueError( + "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm requires --use-replayssm" + ) + if ( + envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" + and self.model_config is not None + and not use_gdn_replayssm + ): + raise ValueError( + "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm currently supports " + "only Qwen3.5 architectures" + ) + if ( + self.cache_config.use_replayssm + and is_gdn_replayssm_architecture + and envs.VLLM_GDN_DECODE_KERNEL != "flashinfer_replayssm" + ): + raise ValueError( + "--use-replayssm on Qwen3.5 requires " + "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm" + ) if not self.cache_config.use_replayssm: self.cache_config.use_kda_recoverssm = False return self + if use_gdn_replayssm: + from vllm.model_executor.layers.mamba.mamba_utils import ( + gdn_replayssm_geometry, + ) + + if ( + self.speculative_config is not None + and self.speculative_config.enable_adaptive_verification + ): + raise ValueError( + "FlashInfer GDN ReplaySSM does not support adaptive verification; " + "set enable_adaptive_verification=False" + ) + if self.cache_config.mamba_cache_mode not in ("none", "align"): + raise ValueError( + "FlashInfer GDN ReplaySSM supports only none and align " + "Mamba cache modes; Qwen3.5 does not support all mode" + ) + if ( + self.cache_config.enable_prefix_caching + and self.cache_config.mamba_cache_mode != "align" + ): + raise ValueError( + "FlashInfer GDN ReplaySSM prefix caching requires " + "--mamba-cache-mode=align" + ) + if self.cache_config.replayssm_buffer_len != 16: + raise ValueError( + "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" + ) + gdn_replayssm_geometry(self.num_speculative_tokens) + if self.model_config is not None: + if self.model_config.dtype != torch.bfloat16: + raise ValueError( + "FlashInfer GDN ReplaySSM requires bfloat16 model dtype" + ) + hf_config = self.model_config.hf_text_config + if ( + getattr(hf_config, "linear_key_head_dim", None) != 128 + or getattr(hf_config, "linear_value_head_dim", None) != 128 + ): + raise ValueError("FlashInfer GDN ReplaySSM requires GDN K=V=128") + if self.cache_config.mamba_cache_dtype not in ("auto", "bfloat16"): + raise ValueError( + "FlashInfer GDN ReplaySSM requires a bfloat16 convolution cache" + ) + if self.cache_config.mamba_ssm_cache_dtype not in ( + "auto", + "bfloat16", + ): + raise ValueError( + "FlashInfer GDN ReplaySSM requires a bfloat16 recurrent state" + ) + if self.parallel_config.pipeline_parallel_size > 1: + raise ValueError( + "FlashInfer GDN ReplaySSM currently requires " + "pipeline_parallel_size=1" + ) + if self.parallel_config.use_ubatching: + raise ValueError( + "FlashInfer GDN ReplaySSM does not support overlapping " + "microbatch execution" + ) + if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": + raise ValueError( + "FlashInfer GDN ReplaySSM requires " + "SGLANG_GDN_WY_STRIDED_QKV=1 before process startup" + ) + if ( + self.kv_transfer_config is not None + and self.kv_transfer_config.is_kv_transfer_instance + ): + raise ValueError( + "--use-replayssm is incompatible with KV connectors " + "(P/D disaggregation, KV cache offload)" + ) + self.cache_config.use_kda_recoverssm = False + # The GDN allowlist above validates this integration's lifecycle; + # the generic SupportsReplaySSM marker alone is not sufficient. + return self + kda_architectures = ( "KimiLinearForCausalLM", "KimiK3ForConditionalGeneration", @@ -4033,6 +4159,15 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": ) return self + def is_gdn_replayssm_enabled(self) -> bool: + """Whether this config selects the FlashInfer GDN replay adapter.""" + return bool( + self.cache_config.use_replayssm + and envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" + and self.model_config is not None + and self.model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES + ) + _current_vllm_config: VllmConfig | None = None _current_prefix: str | None = None diff --git a/vllm/envs.py b/vllm/envs.py index 20de13f59da8..674094968a54 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -133,7 +133,7 @@ VLLM_DISABLED_KERNELS: list[str] = [] VLLM_USE_HW_AGNOSTIC: bool = False VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True - VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda" + VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton", "flashinfer_replayssm"] = "cuda" VLLM_DISABLE_PYNCCL: bool = False VLLM_USE_OINK_OPS: bool = False VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD: bool = False @@ -1281,13 +1281,14 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool( int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1")) ), - # Select the GDN MTP decode implementation. "cuda" uses the fused decode - # kernel where supported and falls back to "triton" otherwise; setting it - # explicitly to "cuda" raises when unsupported. + # Select the GDN decode implementation. "cuda" uses the fused decode kernel + # where supported and falls back to "triton" otherwise; explicitly selecting + # "cuda" raises when unsupported. "flashinfer_replayssm" selects checkpoint + # and replay execution and requires --use-replayssm. "VLLM_GDN_DECODE_KERNEL": env_with_choices( "VLLM_GDN_DECODE_KERNEL", "cuda", - ["cuda", "triton"], + ["cuda", "triton", "flashinfer_replayssm"], case_sensitive=False, ), # Disable pynccl (using torch.distributed instead) diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 249d56685ee7..b453a66f3472 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -14,6 +14,7 @@ from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, + gdn_replayssm_geometry, ) from vllm.model_executor.models.utils import extract_layer_index from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -45,14 +46,47 @@ def __init__( if self.speculative_config else 0 ) + self.use_replayssm = vllm_config.is_gdn_replayssm_enabled() + self.use_flashinfer_replayssm = self.use_replayssm + self.state_alignment = 16 if self.use_flashinfer_replayssm else 1 + self.replayssm_buffer_len = ( + self.cache_config.replayssm_buffer_len if self.use_replayssm else None + ) + self.replayssm_executed_query_width: int | None = None + self.replayssm_ring_slots: int | None = None + if self.use_replayssm: + self.replayssm_executed_query_width, self.replayssm_ring_slots = ( + gdn_replayssm_geometry(self.num_spec) + ) + if self.replayssm_executed_query_width is not None: + # STP executes one token but still uses four staging rows. + packed_width = max(self.replayssm_executed_query_width, 4) + self.register_buffer( + "_replayssm_offsets", + torch.arange(packed_width, dtype=torch.int32), + persistent=False, + ) + self._replayssm_ring_start = torch.empty(0, dtype=torch.int32) + self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32) @property def mamba_type(self) -> MambaAttentionBackendEnum: return MambaAttentionBackendEnum.GDN_ATTN def get_state_dtype(self) -> tuple[torch.dtype, ...]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - self.model_config.dtype, - self.cache_config.mamba_cache_dtype, - self.cache_config.mamba_ssm_cache_dtype, + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + self.model_config.dtype, + self.cache_config.mamba_cache_dtype, + self.cache_config.mamba_ssm_cache_dtype, + ) ) + + if self.use_replayssm: + dtypes = ( + *dtypes, + *MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + self.model_config.dtype + ), + ) + return dtypes diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index bdea28e11acc..949032e95058 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -44,6 +44,10 @@ MambaPrefillCheckpointExporter, ) from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + check_gdn_replayssm_dependencies, + run_gdn_replayssm, +) from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, @@ -677,8 +681,8 @@ def forward_hip( class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): def get_state_shape( self, - ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - return MambaStateShapeCalculator.gated_delta_net_state_shape( + ) -> tuple[tuple[int, ...], ...]: + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( self.tp_size, self.num_k_heads, self.num_v_heads, @@ -688,6 +692,21 @@ def get_state_shape( self.num_spec, ) + if self.use_replayssm: + assert self.replayssm_ring_slots is not None + shapes = ( + *shapes, + *MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + self.tp_size, + self.num_k_heads, + self.num_v_heads, + self.head_k_dim, + self.head_v_dim, + ring_slots=self.replayssm_ring_slots, + ), + ) + return shapes + def __init__( self, config: Qwen3NextConfig, @@ -860,7 +879,19 @@ def __init__( # returns the out_proj input: `out`, or a pre-quantized activation. self._fused_gdn_gated_rms_norm = fused_gdn_gated_rms_norm self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower() - if self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): + if self.use_replayssm: + reason = self._flashinfer_replayssm_unsupported_reason(vllm_config) + if reason is not None: + raise ValueError( + "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm is not " + f"supported: {reason}" + ) + assert self.replayssm_executed_query_width is not None + check_gdn_replayssm_dependencies( + self.replayssm_executed_query_width, + needs_materializer=self.cache_config.mamba_cache_mode == "align", + ) + elif self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): reason = self._fused_gdn_decode_unsupported_reason(vllm_config) if reason is not None: if "VLLM_GDN_DECODE_KERNEL" in os.environ: @@ -889,6 +920,29 @@ def __init__( raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + def _flashinfer_replayssm_unsupported_reason( + self, vllm_config: VllmConfig + ) -> str | None: + conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2] + if self.gqa_interleaved_layout: + return "Qwen3-Next's interleaved GQA layout has not been validated" + if self.head_k_dim != 128 or self.head_v_dim != 128: + return "the FlashInfer kernel requires K=V=128" + if self.num_k_heads % self.tp_size or self.num_v_heads % self.tp_size: + return "tensor-parallel head counts must divide evenly" + if self.num_v_heads % self.num_k_heads: + return "value heads must be an integer multiple of key heads" + if vllm_config.model_config.dtype != torch.bfloat16: + return "model activations must use bfloat16" + if ( + conv_state_dtype != torch.bfloat16 + or recurrent_state_dtype != torch.bfloat16 + ): + return "convolution and recurrent state must use bfloat16" + if not current_platform.has_device_capability(90): + return "the initial integration requires an SM90+ CUDA device" + return None + def get_prefill_checkpoint_alignment(self) -> int | None: return self._checkpoint_alignment @@ -935,7 +989,7 @@ def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]: def _fused_gdn_decode_unsupported_reason( self, vllm_config: VllmConfig ) -> str | None: - conv_state_dtype, recurrent_state_dtype = self.get_state_dtype() + conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2] if ( self.gqa_interleaved_layout or self.head_k_dim != 128 @@ -1326,6 +1380,7 @@ def forward_cuda( use_fused_gdn_spec_decode = ( self.enable_fused_gdn_spec_decode + and not self.use_replayssm and hidden_states.dtype == torch.bfloat16 and self.norm.weight.dtype in (torch.bfloat16, torch.float32) ) @@ -1544,7 +1599,7 @@ def _warmup_prefill_kernels(self, qkv_or_qkvz: torch.Tensor, v_dim: int) -> None dtype = qkv_or_qkvz.dtype num_k_heads = self.num_k_heads // self.tp_size num_v_heads = self.num_v_heads // self.tp_size - _, state_dtype = self.get_state_dtype() + _, state_dtype = self.get_state_dtype()[:2] # All kernels use BT = chunk_size, so a single pass with T = chunk_size # is sufficient to populate every autotuner cache. Mirror the real @@ -1789,6 +1844,7 @@ def _forward_core( if ( self.enable_packed_recurrent_decode + and not self.use_replayssm and attn_metadata.spec_sequence_masks is None and attn_metadata.num_prefills == 0 and attn_metadata.num_decodes > 0 @@ -1860,7 +1916,10 @@ def _forward_core( ], num_accepted_tokens=num_accepted_tokens, query_start_loc=spec_query_start_loc, - max_query_len=spec_state_indices_tensor.size(-1), + max_query_len=( + attn_metadata.replayssm_executed_query_width + or spec_state_indices_tensor.size(-1) + ), validate_data=False, ) @@ -1898,7 +1957,10 @@ def _forward_core( else: mixed_qkv_non_spec = None - query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) + if self.use_replayssm and spec_sequence_masks is not None: + query_spec, key_spec, value_spec = None, None, None + else: + query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) # Split mixed non-spec-decode+prefill to process independently split_non_spec = ( @@ -1961,7 +2023,36 @@ def _forward_core( # 2. Recurrent attention # 2.1: Process the multi-query part - if spec_sequence_masks is not None: + if spec_sequence_masks is not None and self.use_replayssm: + assert mixed_qkv_spec is not None + assert a_spec is not None and b_spec is not None + assert spec_state_indices_tensor is not None + assert spec_query_start_loc is not None + assert attn_metadata.replayssm_output_indices is not None + assert attn_metadata.replayssm_executed_query_width is not None + num_replay_rows = attn_metadata.num_spec_decodes + core_attn_out_spec = run_gdn_replayssm( + mixed_qkv=mixed_qkv_spec, + a=a_spec, + b=b_spec, + A_log=self.A_log, + dt_bias=self.dt_bias, + checkpoint_state=ssm_state, + replayssm_cache=self.kv_cache[2:5], + ring_start=self._replayssm_ring_start, + num_committed=self._replayssm_prev_num_accepted, + query_start_loc=spec_query_start_loc[: num_replay_rows + 1], + state_indices=spec_state_indices_tensor[:num_replay_rows, 0], + output_indices=attn_metadata.replayssm_output_indices, + executed_query_width=(attn_metadata.replayssm_executed_query_width), + num_k_heads=self.num_k_heads // self.tp_size, + num_v_heads=self.num_v_heads // self.tp_size, + head_k_dim=self.head_k_dim, + head_v_dim=self.head_v_dim, + offsets=self._replayssm_offsets, + ).unsqueeze(0) + last_recurrent_state = None + elif spec_sequence_masks is not None: core_attn_out_spec, last_recurrent_state = ( fused_sigmoid_gating_delta_rule_update( A_log=self.A_log, @@ -2103,7 +2194,15 @@ def _forward_core( 0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0) ) elif spec_sequence_masks is not None: - core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0) + if self.use_replayssm: + # FULL graphs may include padded token lanes after all real + # replay rows. Scatter only the gathered real-token outputs. + assert spec_token_indx is not None + core_attn_out.index_copy_( + 0, spec_token_indx.long(), core_attn_out_spec.squeeze(0) + ) + else: + core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0) elif not prefill_in_place: core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0) @@ -2417,7 +2516,8 @@ def _can_use_fused_gdn_mtp_decode( ) -> bool: state_indices = attn_metadata.spec_state_indices_tensor return ( - attn_metadata.spec_sequence_masks is not None + not self.use_replayssm + and attn_metadata.spec_sequence_masks is not None and attn_metadata.num_decodes == 0 and attn_metadata.num_spec_decodes > 0 and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py new file mode 100644 index 000000000000..54fe0d29c039 --- /dev/null +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -0,0 +1,266 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapters for FlashInfer's GDN ReplaySSM decode kernels.""" + +import os +from collections.abc import Callable +from functools import cache + +import torch + +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + +GDN_REPLAY_LOGICAL_WINDOW = 16 + + +@cache +def _check_gdn_replayssm_environment() -> None: + if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": + raise RuntimeError( + "FlashInfer GDN ReplaySSM requires " + "SGLANG_GDN_WY_STRIDED_QKV=1 before importing FlashInfer" + ) + + +@cache +def _load_gdn_replayssm_mtp_kernel() -> Callable[..., torch.Tensor]: + _check_gdn_replayssm_environment() + try: + from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_flush import ( + gated_delta_rule_mtp_ucache_flush, + ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM requires a compatible flashinfer-python " + "build exposing gated_delta_rule_mtp_ucache_flush" + ) from e + return gated_delta_rule_mtp_ucache_flush + + +@cache +def _load_gdn_replayssm_stp_kernel() -> Callable[..., torch.Tensor]: + _check_gdn_replayssm_environment() + try: + from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_stp import ( + gated_delta_rule_stp_ucache_flush, + ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM STP requires a compatible " + "flashinfer-python build exposing " + "gated_delta_rule_stp_ucache_flush" + ) from e + return gated_delta_rule_stp_ucache_flush + + +def check_gdn_replayssm_dependencies( + executed_width: int, needs_materializer: bool +) -> None: + """Check selected FlashInfer exports during GPU worker layer construction.""" + if executed_width == 1: + loader = _load_gdn_replayssm_stp_kernel + api = "gated_delta_rule_stp_ucache_flush" + elif executed_width in (4, 8): + loader = _load_gdn_replayssm_mtp_kernel + api = "gated_delta_rule_mtp_ucache_flush" + else: + raise ValueError(f"Unsupported GDN ReplaySSM executed width: {executed_width}") + mode = f"executed width={executed_width}, cache mode=" + ( + "align" if needs_materializer else "none" + ) + try: + loader() + except ImportError as e: + raise ImportError( + f"FlashInfer GDN ReplaySSM ({mode}) requires {api}: {e}" + ) from e + if needs_materializer: + from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( + _load_gdn_replayssm_materialize, + ) + + try: + _load_gdn_replayssm_materialize() + except ImportError as e: + raise ImportError( + f"FlashInfer GDN ReplaySSM ({mode}) requires " + f"gdn_prefix_materialize: {e}" + ) from e + + +def pack_replayssm_rows( + values: torch.Tensor, + query_start_loc: torch.Tensor, + executed_query_width: int, + offsets: torch.Tensor | None = None, + padding_value: float = 0.0, +) -> torch.Tensor: + """Pack flattened ragged rows into fixed-width rows.""" + num_rows = query_start_loc.numel() - 1 + if num_rows == 0: + return values.new_empty((0, executed_query_width, *values.shape[1:])) + if values.shape[0] == 0: + raise ValueError("cannot pack non-empty ReplaySSM rows from an empty tensor") + if offsets is None: + offsets = torch.arange( + executed_query_width, + dtype=query_start_loc.dtype, + device=query_start_loc.device, + ) + else: + offsets = offsets[:executed_query_width] + row_offsets = offsets.unsqueeze(0).expand(num_rows, -1) + query_lens = torch.diff(query_start_loc).unsqueeze(1) + valid = row_offsets < query_lens + source_indices = query_start_loc[:-1].unsqueeze(1) + row_offsets + source_indices = source_indices.clamp(max=max(values.shape[0] - 1, 0)) + packed = values.index_select(0, source_indices.reshape(-1).long()).view( + num_rows, executed_query_width, *values.shape[1:] + ) + mask = valid.reshape(num_rows, executed_query_width, *([1] * (values.dim() - 1))) + return packed.masked_fill(~mask, padding_value) + + +def run_gdn_replayssm( + *, + mixed_qkv: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + checkpoint_state: torch.Tensor, + replayssm_cache: tuple[torch.Tensor, ...], + ring_start: torch.Tensor, + num_committed: torch.Tensor, + query_start_loc: torch.Tensor, + state_indices: torch.Tensor, + output_indices: torch.Tensor, + executed_query_width: int, + num_k_heads: int, + num_v_heads: int, + head_k_dim: int, + head_v_dim: int, + offsets: torch.Tensor | None = None, + kernel: Callable[..., torch.Tensor] | None = None, +) -> torch.Tensor: + """Pack rows, invoke FlashInfer, and return only real-token outputs. + + STP receives one real row per request. The adapter supplies FlashInfer's + caller-owned padded T=4 staging layout, while vLLM keeps cursor publication + model-wide so every layer observes the same pre-step cursor. + """ + if executed_query_width not in (1, 4, 8): + raise ValueError( + f"FlashInfer GDN ReplaySSM requires executed width 1, 4, or 8, got " + f"{executed_query_width}" + ) + if len(replayssm_cache) != 3: + raise ValueError( + "FlashInfer GDN ReplaySSM requires exactly three auxiliary rings" + ) + if mixed_qkv.dtype != torch.bfloat16 or checkpoint_state.dtype != torch.bfloat16: + raise ValueError( + "FlashInfer GDN ReplaySSM requires bfloat16 inputs and checkpoint state" + ) + + # Native STP's public contract is T=1, but its serving fast path consumes a + # caller-owned zero-padded T=4 buffer. Packing a shared QKV allocation here + # preserves the common token stride required by block-strided cache pools. + packed_width = 4 if executed_query_width == 1 else executed_query_width + packed_qkv = pack_replayssm_rows(mixed_qkv, query_start_loc, packed_width, offsets) + # CuTe requires every input pointer to be 16-byte aligned. Packing the two + # gates together and splitting them can leave ``b`` at a small byte offset + # into the shared allocation when the local head count is not a multiple of + # eight. Give each gate its own aligned allocation instead. + # The MTP kernel advances every padded lane. Make those lanes exact + # recurrent no-ops: a=-inf gives zero decay after softplus(a + dt_bias), + # while zero q/k/v and b give a zero state update. STP has no ragged real + # rows, so only invalid graph-padding requests can be padded here; their + # negative state index skips the kernel CTA. + packed_a = pack_replayssm_rows( + a, + query_start_loc, + packed_width, + offsets, + padding_value=0.0 if executed_query_width == 1 else -torch.inf, + ) + packed_b = pack_replayssm_rows(b, query_start_loc, packed_width, offsets) + local_k_dim = num_k_heads * head_k_dim + local_v_dim = num_v_heads * head_v_dim + q_flat, k_flat, v_flat = packed_qkv.split( + (local_k_dim, local_k_dim, local_v_dim), dim=-1 + ) + q = q_flat.view(packed_qkv.size(0), packed_width, num_k_heads, head_k_dim) + k = k_flat.view_as(q) + v = v_flat.view(packed_qkv.size(0), packed_width, num_v_heads, head_v_dim) + # vLLM reserves block zero for padding, while FlashInfer exits a null-row + # CTA only for a negative index. Gather trackers through the safe padding + # slot, then translate every non-live row to FlashInfer's -1 sentinel. + valid_slots = state_indices > NULL_BLOCK_ID + safe_slots = torch.where(valid_slots, state_indices, NULL_BLOCK_ID).long() + fi_state_indices = torch.where(valid_slots, state_indices, -1) + hist_len = num_committed.index_select(0, safe_slots).masked_fill(~valid_slots, 0) + cache_base = ring_start.index_select(0, safe_slots).masked_fill(~valid_slots, 0) + u_cache, k_cache, g_cache = replayssm_cache + output = torch.zeros( + packed_qkv.size(0), + executed_query_width, + num_v_heads, + head_v_dim, + dtype=mixed_qkv.dtype, + device=mixed_qkv.device, + ) + + if kernel is None: + kernel = ( + _load_gdn_replayssm_stp_kernel() + if executed_query_width == 1 + else _load_gdn_replayssm_mtp_kernel() + ) + common_kwargs = dict( + # These are immutable model parameters. CuTe's DLPack bridge rejects + # tensors whose Parameter wrapper still advertises autograd tracking, + # even while vLLM executes under inference mode. + A_log=A_log.detach(), + a=packed_a, + dt_bias=dt_bias.detach(), + q=q, + k=k, + v=v, + b=packed_b, + initial_state_source=checkpoint_state, + initial_state_indices=fi_state_indices, + use_qk_l2norm_in_kernel=True, + scale=head_k_dim**-0.5, + output=output, + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + hist_len=hist_len, + cache_base=cache_base, + ) + if executed_query_width == 1: + # The STP wrapper self-commits these gathered cursor copies. Do not + # scatter them into the pool here: subsequent GDN layers must consume + # the same pre-step values. The model-wide postprocess publishes the + # equivalent transition once after the target step completes. + kernel( + **common_kwargs, + flush_min=GDN_REPLAY_LOGICAL_WINDOW - 1, + prepadded=True, + ) + else: + kernel( + **common_kwargs, + flush_min=GDN_REPLAY_LOGICAL_WINDOW + 1 - executed_query_width, + restart_hist_on_flush=False, + ) + return output.flatten(0, 1).index_select(0, output_indices.long()) + + +__all__ = [ + "GDN_REPLAY_LOGICAL_WINDOW", + "check_gdn_replayssm_dependencies", + "pack_replayssm_rows", + "run_gdn_replayssm", +] diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 26a3ee785fb8..f5ab1387be57 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -28,6 +28,21 @@ ConvStateLayoutType = Literal["SD", "DS"] +def gdn_replayssm_geometry(num_speculative_tokens: int) -> tuple[int, int]: + """Return GDN's executed query width and physical replay ring slots.""" + if num_speculative_tokens == 0: + return 1, 16 + if 1 <= num_speculative_tokens <= 3: + return 4, 32 + if 4 <= num_speculative_tokens <= 7: + return 8, 32 + raise ValueError( + "FlashInfer GDN ReplaySSM supports 0 through 7 speculative " + "tokens by padding MTP launches to width 4 or 8; got " + f"{num_speculative_tokens}" + ) + + def mamba_state_alignment(vllm_config: "VllmConfig") -> int: # FlashInfer loads state and ring caches with 16-byte cp.async instructions. return 16 if vllm_config.use_flashinfer_replayssm else 1 @@ -137,6 +152,15 @@ def gated_delta_net_state_dtype( model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype ) + @classmethod + def gated_delta_net_replayssm_ring_dtypes( + cls, + model_dtype: ModelDType | torch.dtype, + ) -> tuple[torch.dtype, torch.dtype, torch.dtype]: + """Return FlashInfer GDN ``(u, k, G)`` ring dtypes.""" + activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype) + return (activation_dtype, activation_dtype, torch.float32) + @classmethod def kda_state_dtype( cls, @@ -300,6 +324,25 @@ def gated_delta_net_state_shape( ) return conv_state_shape, temporal_state_shape + @classmethod + def gated_delta_net_replayssm_ring_shapes( + cls, + tp_world_size: int, + num_k_heads: int, + num_v_heads: int, + head_k_dim: int, + head_v_dim: int, + ring_slots: int = 32, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + """Return FlashInfer GDN ``(u, k, G)`` ring shapes.""" + local_k_heads = divide(num_k_heads, tp_world_size) + local_v_heads = divide(num_v_heads, tp_world_size) + return ( + (local_v_heads, ring_slots, head_v_dim), + (local_k_heads, ring_slots, head_k_dim), + (local_v_heads, ring_slots), + ) + @classmethod def kda_state_shape( cls, diff --git a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py index 435b2273673c..55ebe8fe565e 100644 --- a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py +++ b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py @@ -451,6 +451,20 @@ def selective_state_update_replayssm_flashinfer( ) +@cache +def _load_gdn_replayssm_materialize() -> Callable[..., None]: + try: + from flashinfer.gdn_kernels.gdn_prefix_materialize import ( + gdn_prefix_materialize, + ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM prefix caching requires " + "flashinfer.gdn_kernels.gdn_prefix_materialize" + ) from e + return gdn_prefix_materialize + + def initialize_mamba_ssu_backend( mamba_config: MambaConfig, kv_cache_config: KVCacheConfig, diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 3635a637bc77..59880e880106 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -85,7 +85,14 @@ """ MambaStateShapes: TypeAlias = ( - tuple[tuple[int, int]] + tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + ] + | tuple[tuple[int, int]] | tuple[tuple[int, int, int]] | tuple[tuple[int, int], tuple[int, int]] | tuple[tuple[int, int], tuple[int, int, int]] diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 6e370f180da5..7760c3ff3801 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -53,6 +53,7 @@ MambaStateCopyFuncCalculator, MambaStateDtypeCalculator, MambaStateShapeCalculator, + gdn_replayssm_geometry, ) from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -66,12 +67,14 @@ from .interfaces import ( HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsEagle3, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from .qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP @@ -301,6 +304,7 @@ class Qwen3_5ForCausalLMBase( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, ): packed_modules_mapping = { "qkv_proj": [ @@ -386,17 +390,25 @@ def forward( def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) ) + if vllm_config.is_gdn_replayssm_enabled(): + dtypes += MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + vllm_config.model_config.dtype + ) + return dtypes + @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: parallel_config = vllm_config.parallel_config hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size @@ -405,7 +417,7 @@ def get_mamba_state_shape_from_config( if vllm_config.speculative_config else 0 ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, @@ -415,6 +427,18 @@ def get_mamba_state_shape_from_config( num_spec, ) + if vllm_config.is_gdn_replayssm_enabled(): + _, ring_slots = gdn_replayssm_geometry(num_spec) + shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + ring_slots=ring_slots, + ) + return shapes + @classmethod def get_mamba_state_copy_func( cls, @@ -468,7 +492,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): info=Qwen3_5ProcessingInfo, dummy_inputs=Qwen3VLDummyInputsBuilder, ) -class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid): +class Qwen3_5ForConditionalGeneration( + Qwen3VLForConditionalGeneration, IsHybrid, SupportsReplaySSM +): supports_multimodal_pruning = True hf_to_vllm_mapper = ( @@ -607,34 +633,14 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, - ) + ) -> tuple[torch.dtype, ...]: + return Qwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, - ) + ) -> MambaStateShapes: + return Qwen3_5ForCausalLMBase.get_mamba_state_shape_from_config(vllm_config) @classmethod def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 6f233ec1fe22..5413f486c632 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -41,6 +41,7 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup from vllm.model_executor.warmup.qwen_vl_triton_warmup import qwen_vl_triton_warmup from vllm.model_executor.warmup.replayssm_warmup import ( + gdn_replayssm_warmup, replayssm_autotune_warmup, replayssm_lifecycle_warmup, ) @@ -338,6 +339,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): elif has_flashinfer() and current_platform.has_device_capability(90): flashinfer_autotune(worker.model_runner) + # GDN ReplaySSM compiles CuTeDSL artifacts without FlashInfer autotuning. + gdn_replayssm_warmup(worker.model_runner) _warmup_flashinfer_attention(worker) diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index 4830e180d8be..3bb1f31e58e6 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -8,7 +8,9 @@ import torch from vllm.logger import init_logger +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_geometry from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( + _load_gdn_replayssm_materialize, flashinfer_replayssm_autotune_supported, ) @@ -23,7 +25,7 @@ def _replayssm_autotune_kwargs( max_token_prefill_kwargs: dict[str, Any], ) -> tuple[int, dict[str, Any]] | None: config = runner.vllm_config - if not config.use_flashinfer_replayssm: + if not config.use_flashinfer_replayssm or config.is_gdn_replayssm_enabled(): return None if not flashinfer_replayssm_autotune_supported(): logger.info_once( @@ -76,12 +78,10 @@ def _temporary_replayssm_autotune_state( *, prepare_dummy_block_tables: bool = True, ) -> Iterator[None]: - from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 - reset_tensors: list[torch.Tensor] = [] reset_trackers: dict[int, torch.Tensor] = {} for module in runner.get_model().modules(): - if not isinstance(module, MambaMixer2) or not module.use_replayssm: + if not getattr(module, "use_flashinfer_replayssm", False): continue assert module.replayssm_buffer_len is not None ring_start = module._replayssm_ring_start @@ -247,3 +247,115 @@ def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None: ensure_kv_cache_blocks(v2_runner, 1 + max_num_reqs) with _temporary_replayssm_autotune_state(runner, max_num_reqs): runner._dummy_run(**decode_kwargs) + + +def _warm_gdn_replayssm_materializer( + runner: "GPUModelRunner", max_num_reqs: int +) -> None: + """Compile every distinct GDN prefix-materializer specialization.""" + if runner.vllm_config.cache_config.mamba_cache_mode != "align": + return + + materialize = _load_gdn_replayssm_materialize() + warmed: set[tuple[Any, ...]] = set() + for module in runner.get_model().modules(): + if not getattr(module, "use_flashinfer_replayssm", False) or getattr( + module, "replayssm_executed_query_width", None + ) not in (1, 4, 8): + continue + state = module.kv_cache[1] + u_cache, k_cache, g_cache = module.kv_cache[2:5] + signature = ( + state.device, + state.dtype, + tuple(state.shape), + tuple(state.stride()), + tuple(k_cache.shape), + tuple(k_cache.stride()), + tuple(u_cache.shape), + tuple(u_cache.stride()), + tuple(g_cache.shape), + tuple(g_cache.stride()), + ) + if signature in warmed: + continue + warmed.add(signature) + inactive = torch.full( + (max_num_reqs,), -1, dtype=torch.int32, device=state.device + ) + zeros = torch.zeros_like(inactive) + materialize( + state=state, + src_slots=inactive, + dst_slots=inactive, + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + cache_base=zeros, + count=inactive, + active_request_indices=inactive, + num_active=torch.zeros(1, dtype=torch.int32, device=state.device), + ) + + +def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: + """Compile GDN replay for every intended uniform-decode graph bucket.""" + config = runner.vllm_config + if not config.is_gdn_replayssm_enabled(): + return + from vllm.v1.worker.extensible_kv_cache import ( + ensure_kv_cache_blocks, + num_committable_kv_blocks, + ) + + v2_runner: Any = runner + executed_query_width, _ = gdn_replayssm_geometry(config.num_speculative_tokens) + query_len = config.num_speculative_tokens + 1 + assert query_len <= executed_query_width + max_num_reqs = min( + runner.scheduler_config.max_num_seqs, + runner.max_num_tokens // query_len, + num_committable_kv_blocks(v2_runner) - 1, + ) + if max_num_reqs <= 0: + raise ValueError( + "FlashInfer GDN ReplaySSM warmup requires a non-padding state slot" + ) + if config.use_v2_model_runner: + manager = v2_runner.cudagraph_manager + capture_reqs = { + desc.num_reqs + for descs in manager._capture_descs.values() + for desc in descs + if desc.uniform_token_count == query_len + and desc.num_reqs is not None + and 0 < desc.num_reqs <= max_num_reqs + } + else: + capture_reqs = { + size // query_len + for size in runner.cudagraph_batch_sizes + if size % query_len == 0 and 0 < size // query_len <= max_num_reqs + } + if not capture_reqs: + capture_reqs.add(max_num_reqs) + + decode_kwargs: dict[str, Any] = { + "uniform_decode": True, + "skip_eplb": True, + "is_profile": True, + "randomize_inputs": True, + } + if config.use_v2_model_runner: + decode_kwargs["valid_dummy_state_slots"] = True + else: + decode_kwargs.update( + allow_microbatching=False, + force_attention=True, + profile_seq_lens=query_len + 1, + ) + ensure_kv_cache_blocks(v2_runner, 1 + max_num_reqs) + with _temporary_replayssm_autotune_state(runner, max_num_reqs): + for num_reqs in sorted(capture_reqs): + runner._dummy_run(num_tokens=num_reqs * query_len, **decode_kwargs) + _warm_gdn_replayssm_materializer(runner, max_num_reqs) diff --git a/vllm/models/qwen3_5/nvidia/model.py b/vllm/models/qwen3_5/nvidia/model.py index be483f5a88dd..9b6101e8659e 100644 --- a/vllm/models/qwen3_5/nvidia/model.py +++ b/vllm/models/qwen3_5/nvidia/model.py @@ -62,8 +62,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, ) from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( nvfp4_weight_padding_bytes, @@ -81,15 +79,20 @@ EagleModelMixin, HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsEagle3, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP +from vllm.model_executor.models.qwen3_5 import ( + Qwen3_5ForCausalLMBase as CommonQwen3_5ForCausalLMBase, +) from vllm.model_executor.models.qwen3_5 import ( Qwen3_5MoeProcessingInfo, Qwen3_5ProcessingInfo, @@ -633,6 +636,7 @@ class Qwen3_5ForCausalLMBase( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, ): packed_modules_mapping = { "qkv_proj": [ @@ -719,33 +723,17 @@ def forward( def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config( + vllm_config ) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, + ) -> MambaStateShapes: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_shape_from_config( + vllm_config ) @classmethod @@ -801,7 +789,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): info=Qwen3_5ProcessingInfo, dummy_inputs=Qwen3VLDummyInputsBuilder, ) -class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid): +class Qwen3_5ForConditionalGeneration( + Qwen3VLForConditionalGeneration, IsHybrid, SupportsReplaySSM +): supports_multimodal_pruning = True hf_to_vllm_mapper = ( @@ -923,33 +913,17 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config( + vllm_config ) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, + ) -> MambaStateShapes: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_shape_from_config( + vllm_config ) @classmethod diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index e5fbf23563fd..8dde6c03db1b 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -448,6 +448,11 @@ class CommonAttentionMetadata: (num_computed_tokens < num_prompt_tokens). Used by some backends to distinguish actual decodes from short extends.""" + is_last_prefill: torch.Tensor | None = None + """(batch_size,) bool tensor: True if this scheduled chunk reaches the end + of a request's prompt. Used with ``is_prefilling`` by recurrent backends + that can safely route only the final one-token prompt chunk as decode.""" + seq_lens_cpu_upper_bound: torch.Tensor | None = None """(batch_size,) CPU upper bound on seq_lens. Precise for prefill rows and for all rows outside async spec decode; optimistic for async-spec @@ -553,6 +558,7 @@ def unpadded( seq_lens_cpu_upper_bound=maybe_slice_reqs(self.seq_lens_cpu_upper_bound), is_prefilling=maybe_slice_reqs(self.is_prefilling), req_idx=maybe_slice_reqs(self.req_idx), + is_last_prefill=maybe_slice_reqs(self.is_last_prefill), rswa_prefix_lens=maybe_slice_reqs(self.rswa_prefix_lens), ) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index f28b3e42adb3..0cfbf2bf1ba6 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -14,6 +14,7 @@ MambaPrefillCheckpointBuilder, MambaPrefillCheckpointMetadata, ) +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_geometry from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -121,6 +122,11 @@ class GDNAttentionMetadata: num_accepted_tokens: torch.Tensor | None = None # shape: [batch,] uniform_spec_sequence_length: int | None = None # None for ragged batches + # FlashInfer GDN ReplaySSM packs real-token rows into T=1/4/8 launches and + # gathers these dense output positions back afterward. + replayssm_output_indices: torch.Tensor | None = None + replayssm_executed_query_width: int | None = None + # Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices) chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -192,6 +198,10 @@ def __init__( else: self.num_spec = 0 self.use_spec_decode: bool = self.num_spec > 0 + self.use_gdn_replayssm = vllm_config.is_gdn_replayssm_enabled() + self.replayssm_executed_query_width = ( + gdn_replayssm_geometry(self.num_spec)[0] if self.use_gdn_replayssm else None + ) self._init_reorder_batch_threshold(1, self.use_spec_decode) self.use_full_cuda_graph: bool = ( @@ -219,8 +229,9 @@ def __init__( self.compilation_config.max_cudagraph_capture_size, ) + self.state_table_width = 1 if self.use_gdn_replayssm else self.num_spec + 1 self.spec_state_indices_tensor: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs, self.num_spec + 1), + (self.decode_cudagraph_max_bs, self.state_table_width), dtype=torch.int32, device=device, ) @@ -259,6 +270,11 @@ def __init__( dtype=torch.int32, device=device, ) + self.replayssm_output_indices: torch.Tensor | None = None + if self.use_gdn_replayssm: + self.replayssm_output_indices = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device + ) @staticmethod def _check_chunk_metadata_override( @@ -354,7 +370,49 @@ def build( # type: ignore[override] uniform_spec_sequence_length = None spec_sequence_masks_cpu: torch.Tensor | None = None - if not self.use_spec_decode or num_decode_draft_tokens_cpu is None: + if self.use_gdn_replayssm: + if ( + m.is_prefilling is None + or m.is_last_prefill is None + or m.seq_lens_cpu_upper_bound is None + ): + raise ValueError( + "FlashInfer GDN ReplaySSM requires prefill, final-prompt, and " + "sequence-length metadata" + ) + query_lens_cpu_all = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + decode_rows = query_lens_cpu_all == 1 + if num_decode_draft_tokens_cpu is not None: + decode_rows |= (num_decode_draft_tokens_cpu >= 0) & ( + query_lens_cpu_all == num_decode_draft_tokens_cpu + 1 + ) + assert self.replayssm_executed_query_width is not None + replayssm_prefilling = m.is_prefilling & ~( + m.is_last_prefill + & (m.seq_lens_cpu_upper_bound > query_lens_cpu_all) + & decode_rows + ) + replay_width = self.replayssm_executed_query_width + spec_sequence_masks_cpu = ( + ~replayssm_prefilling + & (query_lens_cpu_all > 0) + & (query_lens_cpu_all <= replay_width) + ) + num_spec_decodes = int(spec_sequence_masks_cpu.sum().item()) + if num_spec_decodes == 0: + spec_sequence_masks = None + spec_sequence_masks_cpu = None + else: + spec_sequence_masks = async_tensor_h2d( + spec_sequence_masks_cpu, device=query_start_loc.device + ) + if num_accepted_tokens is None: + num_accepted_tokens = torch.ones( + m.num_reqs, + dtype=torch.int32, + device=query_start_loc.device, + ) + elif not self.use_spec_decode or num_decode_draft_tokens_cpu is None: spec_sequence_masks = None num_spec_decodes = 0 else: @@ -372,6 +430,7 @@ def build( # type: ignore[override] spec_sequence_masks_cpu, device=query_start_loc.device ) + replayssm_output_indices = None if spec_sequence_masks is None: # V2 already excludes prefills from full decode graphs via # has_prefill. Classify first chunks as prefills to mask recycled @@ -389,7 +448,9 @@ def build( # type: ignore[override] no_prior_state = torch.zeros_like(no_prior_state) num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( - m.replace(is_prefilling=no_prior_state), + m + if self.use_gdn_replayssm + else m.replace(is_prefilling=no_prior_state), decode_threshold=1, treat_short_extends_as_decodes=False, ) @@ -448,9 +509,13 @@ def build( # type: ignore[override] num_decode_tokens = 0 if num_prefills == 0 and num_decodes == 0: - spec_token_size = min( - num_spec_decodes * (self.num_spec + 1), - query_start_loc_cpu[-1].item(), + spec_token_size = ( + query_start_loc_cpu[-1].item() + if self.use_gdn_replayssm + else min( + num_spec_decodes * (self.num_spec + 1), + query_start_loc_cpu[-1].item(), + ) ) spec_token_indx = torch.arange( spec_token_size, @@ -463,7 +528,7 @@ def build( # type: ignore[override] # Padded sequences trail the spec decodes, so slice them off # rather than gather with the host mask (an H2D copy + a kernel). spec_state_indices_tensor = block_table_tensor[ - :num_spec_decodes, : self.num_spec + 1 + :num_spec_decodes, : self.state_table_width ] non_spec_state_indices_tensor = None # Padded sequences are always at the back, so the first @@ -484,7 +549,7 @@ def build( # type: ignore[override] spec_token_indx = index[num_non_spec_tokens:] spec_state_indices_tensor = block_table_tensor[ - spec_sequence_masks_cpu, : self.num_spec + 1 + spec_sequence_masks_cpu, : self.state_table_width ] non_spec_state_indices_tensor = block_table_tensor[ non_spec_sequence_masks_cpu, 0 @@ -525,6 +590,21 @@ def build( # type: ignore[override] num_accepted_tokens = num_accepted_tokens[:num_spec_decodes] else: num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] + if self.use_gdn_replayssm: + assert self.replayssm_executed_query_width is not None + replay_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu] + row_offsets = torch.arange( + self.replayssm_executed_query_width, dtype=torch.int32 + ).unsqueeze(0) + dense_indices = ( + torch.arange(num_spec_decodes, dtype=torch.int32).unsqueeze(1) + * self.replayssm_executed_query_width + + row_offsets + ) + replayssm_output_indices = async_tensor_h2d( + dense_indices[row_offsets < replay_query_lens_cpu.unsqueeze(1)], + device=query_start_loc.device, + ) chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -672,6 +752,15 @@ def build( # type: ignore[override] num_accepted_tokens = self.num_accepted_tokens[:batch_size] num_accepted_tokens[num_spec_decodes:].fill_(1) + if self.use_gdn_replayssm: + assert replayssm_output_indices is not None + assert self.replayssm_output_indices is not None + output_count = replayssm_output_indices.numel() + self.replayssm_output_indices[:output_count].copy_( + replayssm_output_indices, non_blocking=True + ) + replayssm_output_indices = self.replayssm_output_indices[:output_count] + if self._stage_decode(num_prefills, num_decodes, num_spec_decodes): self.non_spec_state_indices_tensor[:num_decodes].copy_( non_spec_state_indices_tensor, non_blocking=True @@ -715,6 +804,8 @@ def build( # type: ignore[override] non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, uniform_spec_sequence_length=uniform_spec_sequence_length, + replayssm_output_indices=replayssm_output_indices, + replayssm_executed_query_width=self.replayssm_executed_query_width, nums_dict=nums_dict, batch_ptr=batch_ptr, token_chunk_offset_ptr=token_chunk_offset_ptr, @@ -773,9 +864,9 @@ def update_block_table( prefill_indices = non_spec_indices[m.num_decodes :] elif m.num_prefills == 0: # Same as build(): padded sequences trail the spec decodes. - spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1] + spec_indices = blk_table[: m.num_spec_decodes, : self.state_table_width] else: - spec_indices = blk_table[masks, : self.num_spec + 1] + spec_indices = blk_table[masks, : self.state_table_width] non_spec_indices = prefill_indices = blk_table[~masks, 0] if self._stage_spec_decode( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 5286d9008d06..85c75778138e 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1430,6 +1430,12 @@ def gather_batch_req_state( if num_draft_tokens_np is not None: num_new_tokens = num_new_tokens - num_draft_tokens_np one_token_ext = (num_new_tokens == 1) & (num_computed_prefill_tokens_np > 0) + if self.vllm_config.is_gdn_replayssm_enabled(): + # Intermediate GDN chunks must update the canonical state before + # a later multi-token prefill reads it. + one_token_ext &= ( + num_computed_prefill_tokens_np + num_new_tokens >= prefill_len_np + ) prefill_runs_as_decode_np = is_prefilling_np & one_token_ext decode_graph_eligible = bool( (prefill_runs_as_decode_np | ~is_prefilling_np).all() diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index d97672d16326..af708717e94c 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -12,7 +12,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d -from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder +from vllm.v1.attention.backends.gdn_attn import ( + GDNAttentionMetadata, + GDNAttentionMetadataBuilder, +) from vllm.v1.attention.backends.mamba2_attn import ( Mamba2AttentionMetadata, Mamba2AttentionMetadataBuilder, @@ -45,13 +48,17 @@ @dataclass class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): is_prefilling: torch.Tensor + is_last_prefill: torch.Tensor num_accepted_tokens: torch.Tensor | None = None num_decode_draft_tokens_cpu: torch.Tensor | None = None def get_extra_common_attn_kwargs( self, kv_cache_group_id: int, num_reqs: int ) -> dict[str, Any]: - return {"is_prefilling": self.is_prefilling[:num_reqs]} + return { + "is_prefilling": self.is_prefilling[:num_reqs], + "is_last_prefill": self.is_last_prefill[:num_reqs], + } def get_extra_attn_kwargs( self, attn_metadata_builder: Any, num_reqs: int @@ -98,6 +105,7 @@ def __init__( # running state_idx are kept GPU-resident. self._align_mode = self.cache_config.mamba_cache_mode == "align" self._use_flashinfer_replayssm = vllm_config.use_flashinfer_replayssm + self._use_gdn_replayssm = vllm_config.is_gdn_replayssm_enabled() if self._use_flashinfer_replayssm: self._is_prefilling_gpu = torch.zeros( self.max_num_reqs, dtype=torch.bool, device=self.device @@ -290,6 +298,15 @@ def prepare_attn( is_prefilling_np = is_prefilling_np & ~input_batch.prefill_runs_as_decode_np is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") is_prefilling[: input_batch.num_reqs] = torch.from_numpy(is_prefilling_np) + is_last_prefill = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") + is_last_prefill[: input_batch.num_reqs] = torch.from_numpy( + input_batch.is_prefilling_np + & ( + input_batch.num_computed_prefill_tokens_np + + input_batch.num_scheduled_tokens + >= input_batch.prefill_len_np + ) + ) # During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens # are created by attn_metadata_builder.build_for_cudagraph_capture, so we only # compute them during actual (non-capture) forward execution. @@ -342,6 +359,7 @@ def prepare_attn( mamba_attn_metadata = MambaHybridAttnMetadata( is_prefilling=is_prefilling, + is_last_prefill=is_last_prefill, num_accepted_tokens=num_accepted_tokens, num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) @@ -366,20 +384,34 @@ def prepare_attn( ) if self._use_flashinfer_replayssm: self._replayssm_query_start_loc = None + metadata_type = ( + GDNAttentionMetadata + if self._use_gdn_replayssm + else Mamba2AttentionMetadata + ) mamba_metadata = next( ( metadata for metadata in attn_metadata.values() - if isinstance(metadata, Mamba2AttentionMetadata) + if isinstance(metadata, metadata_type) ), None, ) # Autotune dummy runs and draft capture can omit Mamba metadata. if mamba_metadata is not None: self._replayssm_query_start_loc = input_batch.query_start_loc - replayssm_prefilling = ( - torch.arange(num_reqs, device="cpu") >= mamba_metadata.num_decodes - ) + if ( + isinstance(mamba_metadata, GDNAttentionMetadata) + and mamba_metadata.spec_sequence_masks_cpu is not None + ): + replayssm_prefilling = ~mamba_metadata.spec_sequence_masks_cpu[ + :num_reqs + ] + else: + replayssm_prefilling = ( + torch.arange(num_reqs, device="cpu") + >= mamba_metadata.num_decodes + ) async_tensor_h2d( replayssm_prefilling, out=self._is_prefilling_gpu[:num_reqs] ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index c2de73d98dee..70599aed1787 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -128,7 +128,10 @@ AttentionType, CommonAttentionMetadata, ) -from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder +from vllm.v1.attention.backends.gdn_attn import ( + GDNAttentionMetadata, + GDNAttentionMetadataBuilder, +) from vllm.v1.attention.backends.linear_attn import ( BailingLinearAttentionMetadataBuilder, ) @@ -2427,9 +2430,13 @@ def _get_block_table(kv_cache_gid: int): # Used by mamba backends to distinguish actual decodes from # short extends. is_prefilling = num_computed_tokens_cpu < num_prompt_tokens_cpu + is_last_prefill = is_prefilling & ( + seq_lens_cpu_upper_bound >= num_prompt_tokens_cpu + ) # Zero out padded rows so stale data from condense() doesn't # misclassify padding as prefill in CUDA graph mode. is_prefilling[num_reqs:] = False + is_last_prefill[num_reqs:] = False # Compute mm_prefix bidirectional ranges before building # attention metadata so builders handle them during build(). @@ -2469,6 +2476,7 @@ def _get_block_table(kv_cache_gid: int): slot_mapping=slot_mapping_gid_0, causal=True, is_prefilling=is_prefilling, + is_last_prefill=is_last_prefill, positions=self.positions[:num_tokens_padded], mm_req_doc_ranges=req_doc_ranges, rswa_prefix_lens=rswa_prefix_lens, @@ -4198,6 +4206,7 @@ def execute_model( if mamba_bufs is not None and mamba_bufs.postprocess_align is not None: mamba_ctx = mamba_bufs.postprocess_align num_mamba_decodes = None + gdn_prefilling = None if self._use_flashinfer_replayssm: assert isinstance(attn_metadata, dict) mamba_group = self.kv_cache_config.kv_cache_groups[ @@ -4205,8 +4214,15 @@ def execute_model( ] # Cache groups share the request split, but own distinct slots. mamba_metadata = attn_metadata[mamba_group.layer_names[0]] - assert isinstance(mamba_metadata, Mamba2AttentionMetadata) - num_mamba_decodes = mamba_metadata.num_decodes + if isinstance(mamba_metadata, GDNAttentionMetadata): + replay_rows = mamba_metadata.spec_sequence_masks_cpu + if replay_rows is not None: + gdn_prefilling = ~replay_rows[:num_reqs] + else: + num_mamba_decodes = mamba_metadata.num_decodes + else: + assert isinstance(mamba_metadata, Mamba2AttentionMetadata) + num_mamba_decodes = mamba_metadata.num_decodes mamba_utils.stage_postprocess_inputs_to_gpu( mamba_ctx, scheduler_output, @@ -4214,6 +4230,7 @@ def execute_model( num_reqs, self.requests, num_decodes=num_mamba_decodes, + is_prefilling=gdn_prefilling, ) ( diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 1bc2a33a8859..9ab10f2b7052 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -798,13 +798,17 @@ def _get_mamba_gpu_state_copy_funcs( state_copy_funcs = copy_funcs[spec.mamba_type] if isinstance(layer, MambaBase) and layer.use_flashinfer_replayssm: if ( - spec.mamba_type != MambaAttentionBackendEnum.MAMBA2 + spec.mamba_type + not in ( + MambaAttentionBackendEnum.MAMBA2, + MambaAttentionBackendEnum.GDN_ATTN, + ) or len(spec.shapes) != 5 or len(spec.dtypes) != 5 or state_copy_funcs != (get_conv_copy_spec, get_temporal_copy_spec) ): raise ValueError( - "FlashInfer ReplaySSM requires two canonical Mamba2 " + "FlashInfer ReplaySSM requires two canonical Mamba2 or GDN " "states and three ring states" ) state_copy_funcs += (get_temporal_copy_spec,) * 3 @@ -1064,7 +1068,7 @@ def _populate_metadata( if is_flashinfer_replayssm: if len(kv_caches) != len(state_copy_funcs): raise ValueError( - "FlashInfer ReplaySSM requires five bound Mamba2 states" + "FlashInfer ReplaySSM requires five bound recurrent states" ) mixers.append(attention) if len(kv_caches) < len(state_copy_funcs): @@ -1773,6 +1777,7 @@ def stage_postprocess_inputs_to_gpu( requests: dict[str, CachedRequestState], *, num_decodes: int | None = None, + is_prefilling: torch.Tensor | None = None, ) -> None: """Stage the per-request decisions consumed after token acceptance. @@ -1801,7 +1806,10 @@ def stage_postprocess_inputs_to_gpu( ctx.num_scheduled_tokens_buf.copy_to_gpu(num_reqs) ctx.num_computed_tokens_buf.copy_to_gpu(num_reqs) ctx.num_draft_tokens_buf.copy_to_gpu(num_reqs) - if num_decodes is not None: + if is_prefilling is not None: + ctx.is_prefilling_buf.np[:num_reqs] = is_prefilling.numpy() + ctx.is_prefilling_buf.copy_to_gpu(num_reqs) + elif num_decodes is not None: # Full decode graphs can include padded rows after the real requests. num_decodes = min(num_decodes, num_reqs) ctx.is_prefilling_buf.np[:num_decodes] = False diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index 4570b3413a8a..af015ffa2a8e 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -10,8 +10,12 @@ import torch from vllm.model_executor.layers.mamba.abstract import MambaBase +from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( + _load_gdn_replayssm_materialize, +) from vllm.model_executor.models.utils import extract_layer_index from vllm.triton_utils import tl, triton +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec @@ -174,6 +178,8 @@ def _postprocess_replayssm_kernel( HAS_IDX_MAPPING: tl.constexpr, MATERIALIZE_PREFIXES: tl.constexpr, LIVE_COL_IS_ZERO: tl.constexpr, + EXECUTED_QUERY_WIDTH: tl.constexpr, + GDN_STP: tl.constexpr, ) -> None: """Commit a completed step and prepare an optional prefix snapshot.""" batch_idx = tl.program_id(0) @@ -213,7 +219,8 @@ def _postprocess_replayssm_kernel( NUM_COMPUTED_IS_POST_STEP, computed - query_len, computed ) # Staged from forward metadata using Mamba attention's classification. - prefilling = tl.load(is_prefilling + batch_idx) + valid_query = query_len > 0 + prefilling = tl.load(is_prefilling + batch_idx) & valid_query accepted = tl.maximum(tl.load(num_accepted_tokens + req_idx), 1) # Derive this request's pre/post-step positions from ReplaySSM metadata. @@ -283,26 +290,45 @@ def _postprocess_replayssm_kernel( if wants_materialize & (live_col != dst_col): # Prefill already wrote canonical state; only another slot needs a copy. tl.store(plan_flush_count + batch_idx, 0, mask=writes_plan) - elif valid_live: + elif valid_live & valid_query: old_start = tl.load(tracker_start + live_slot) old_committed = tl.load(tracker_committed + live_slot) - checkpointed = old_committed + query_len > LOGICAL_WINDOW - next_start = tl.where( - checkpointed, - (old_start + old_committed) % RING_BUFFER_LEN, - old_start, - ) - next_committed = tl.where(checkpointed, accepted, old_committed + accepted) + if GDN_STP: + # FlashInfer's GDN STP fold absorbs the current token into the + # checkpoint at P == L - 1 and uses a flat 16-slot ring. Layer + # forwards mutate gathered cursor copies; publish the equivalent + # group-owned transition once here. + checkpointed = old_committed >= LOGICAL_WINDOW - 1 + next_start = 0 + next_committed = tl.where(checkpointed, 0, old_committed + 1) + else: + executed_query_len = query_len + if EXECUTED_QUERY_WIDTH > 0: + executed_query_len = EXECUTED_QUERY_WIDTH + checkpointed = old_committed + executed_query_len > LOGICAL_WINDOW + next_start = tl.where( + checkpointed, + (old_start + old_committed) % RING_BUFFER_LEN, + old_start, + ) + next_committed = tl.where(checkpointed, accepted, old_committed + accepted) tl.store(tracker_start + live_slot, next_start) tl.store(tracker_committed + live_slot, next_committed) if wants_materialize: tl.store(plan_ring_start + batch_idx, next_start, mask=writes_plan) - tl.store( - plan_flush_count + batch_idx, - accept_token_bias + 1 + tl.where(checkpointed, 0, old_committed), - mask=writes_plan, - ) + if GDN_STP: + # On a fold, state already contains the current token and an + # exact count=0 copy publishes the boundary. Otherwise replay + # the old history plus the one accepted STP token. + materialize_count = tl.where( + checkpointed, 0, old_committed + accept_token_bias + 1 + ) + else: + materialize_count = ( + accept_token_bias + 1 + tl.where(checkpointed, 0, old_committed) + ) + tl.store(plan_flush_count + batch_idx, materialize_count, mask=writes_plan) # The published destination is canonical and therefore has no live replay. # When dst_slot aliases live_slot, these stores intentionally supersede the @@ -315,6 +341,7 @@ def _postprocess_replayssm_kernel( def _compact_replayssm_requests_kernel( plan_flush_count, active_request_indices, + num_active_ptr, num_reqs, MAX_NUM_REQS: tl.constexpr, BLOCK_SIZE: tl.constexpr, @@ -328,6 +355,7 @@ def _compact_replayssm_requests_kernel( active_i32 = active.to(tl.int32) output_offsets = tl.cumsum(active_i32, axis=0) - 1 num_active = tl.sum(active_i32, axis=0) + tl.store(num_active_ptr, num_active) tl.store( active_request_indices + output_offsets, offsets, @@ -367,17 +395,20 @@ class ReplaySSMModelContext: src_slots_ptrs: torch.Tensor dst_slots_ptrs: torch.Tensor num_layers: torch.Tensor - materialize_tables: _ReplaySSMMaterializeTables + materialize_tables: _ReplaySSMMaterializeTables | None src_slots: torch.Tensor dst_slots: torch.Tensor plan_ring_start: torch.Tensor plan_flush_count: torch.Tensor active_request_indices: torch.Tensor + num_active: torch.Tensor max_num_reqs: int mamba_block_size: int logical_window: int ring_buffer_len: int materialize_prefixes: bool + executed_query_width: int + mamba_type: MambaAttentionBackendEnum @classmethod def create( @@ -388,6 +419,7 @@ def create( mixers = [mixer for group, _, _ in grouped for mixer in group] first = mixers[0] first_spec = grouped[0][1] + mamba_type = first_spec.mamba_type # Homogeneous groups share cache-hit lookup, logical columns, and replay # transitions. The scheduler still assigns each group its own slots. assert all(spec == first_spec for _, spec, _ in grouped), ( @@ -396,12 +428,48 @@ def create( geometry = [(state.shape[1:], state.dtype) for state in first.kv_cache[1:]] assert all( [(state.shape[1:], state.dtype) for state in mixer.kv_cache[1:]] == geometry - and mixer.A.dtype == first.A.dtype + and ( + mamba_type == MambaAttentionBackendEnum.GDN_ATTN + or mixer.A.dtype == first.A.dtype + ) and mixer.replayssm_buffer_len == first.replayssm_buffer_len for mixer in mixers ), "FlashInfer ReplaySSM requires equal materialization geometry" device = first.kv_cache[1].device + logical_window = int(first.replayssm_buffer_len) + executed_query_width = int( + getattr(first, "replayssm_executed_query_width", 0) or 0 + ) + ring_buffer_len = first.kv_cache[2].size(2) + if mamba_type == MambaAttentionBackendEnum.GDN_ATTN: + if logical_window != 16 or executed_query_width not in (1, 4, 8): + raise ValueError( + "FlashInfer GDN ReplaySSM requires logical window 16 and " + "executed query width 1, 4, or 8; got " + f"window={logical_window}, width={executed_query_width}" + ) + expected_ring_buffer_len = 16 if executed_query_width == 1 else 32 + if ring_buffer_len != expected_ring_buffer_len: + raise ValueError( + "FlashInfer GDN ReplaySSM ring depth does not match its " + f"executed width: width={executed_query_width} requires " + f"{expected_ring_buffer_len} slots, got {ring_buffer_len}" + ) + for mixer in mixers[1:]: + mixer_width = int( + getattr(mixer, "replayssm_executed_query_width", 0) or 0 + ) + mixer_ring_len = mixer.kv_cache[2].size(2) + if ( + int(mixer.replayssm_buffer_len) != logical_window + or mixer_width != executed_query_width + or mixer_ring_len != ring_buffer_len + ): + raise ValueError( + "all GDN ReplaySSM layers in a cache group must share " + "the same logical window, executed width, and ring depth" + ) block_tables = [table for _, _, table in grouped] src_slots = torch.full( (len(mixers), max_num_reqs), @@ -415,7 +483,30 @@ def create( layer_offsets.append(layer_offsets[-1] + len(group)) # Each pointer selects a group's first row in the flat [layer, batch] # allocation. Different group sizes need no padding or per-step concat. - zero_table = torch.zeros(len(mixers), dtype=torch.int64, device=device) + materialize_tables = None + if mamba_type != MambaAttentionBackendEnum.GDN_ATTN: + zero_table = torch.zeros(len(mixers), dtype=torch.int64, device=device) + materialize_tables = _ReplaySSMMaterializeTables( + state_ptrs=_cuda_i64_ptrs([m.kv_cache[1] for m in mixers]), + state_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[1] for m in mixers] + ), + x_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[2] for m in mixers]), + x_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[2] for m in mixers] + ), + B_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[4] for m in mixers]), + B_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[4] for m in mixers] + ), + dt_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[3] for m in mixers]), + dt_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[3] for m in mixers] + ), + A_ptrs=_cuda_i64_ptrs([m.A for m in mixers]), + state_scale_ptrs=zero_table, + state_scale_slot_strides=zero_table, + ) return cls( mixers=mixers, block_tables=block_tables, @@ -438,27 +529,7 @@ def create( dtype=torch.int32, device=device, ), - materialize_tables=_ReplaySSMMaterializeTables( - state_ptrs=_cuda_i64_ptrs([m.kv_cache[1] for m in mixers]), - state_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[1] for m in mixers] - ), - x_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[2] for m in mixers]), - x_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[2] for m in mixers] - ), - B_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[4] for m in mixers]), - B_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[4] for m in mixers] - ), - dt_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[3] for m in mixers]), - dt_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[3] for m in mixers] - ), - A_ptrs=_cuda_i64_ptrs([m.A for m in mixers]), - state_scale_ptrs=zero_table, - state_scale_slot_strides=zero_table, - ), + materialize_tables=materialize_tables, src_slots=src_slots, dst_slots=dst_slots, plan_ring_start=torch.zeros(max_num_reqs, dtype=torch.int32, device=device), @@ -468,6 +539,9 @@ def create( active_request_indices=torch.full( (max_num_reqs,), -1, dtype=torch.int32, device=device ), + num_active=torch.zeros(1, dtype=torch.int32, device=device), + executed_query_width=executed_query_width, + mamba_type=mamba_type, max_num_reqs=max_num_reqs, mamba_block_size=first_spec.block_size, logical_window=int(first.replayssm_buffer_len), @@ -476,7 +550,11 @@ def create( ) def _materialize(self) -> None: - """Publish all canonical prefix snapshots in one FlashInfer call.""" + """Publish canonical prefix snapshots with the selected backend.""" + if self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: + self.materialize_gdn(_load_gdn_replayssm_materialize()) + return + assert self.materialize_tables is not None first = self.mixers[0] mamba_config = first.mamba_config rand_seed = None @@ -507,6 +585,23 @@ def _materialize(self) -> None: philox_rounds=philox_rounds, ) + def materialize_gdn(self, materialize_fn: Callable[..., None]) -> None: + """Publish one GDN recurrent-state snapshot per layer.""" + for layer_idx, mixer in enumerate(self.mixers): + u_cache, k_cache, g_cache = mixer.kv_cache[2:5] + materialize_fn( + state=mixer.kv_cache[1], + src_slots=self.src_slots[layer_idx], + dst_slots=self.dst_slots[layer_idx], + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + cache_base=self.plan_ring_start, + count=self.plan_flush_count, + active_request_indices=self.active_request_indices, + num_active=self.num_active, + ) + def reset_new_slots( self, *, @@ -572,12 +667,18 @@ def postprocess( HAS_IDX_MAPPING=idx_mapping is not None, MATERIALIZE_PREFIXES=self.materialize_prefixes, LIVE_COL_IS_ZERO=live_cols is None, + EXECUTED_QUERY_WIDTH=self.executed_query_width, + GDN_STP=( + self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN + and self.executed_query_width == 1 + ), ) if not self.materialize_prefixes: return _compact_replayssm_requests_kernel[(1,)]( self.plan_flush_count, self.active_request_indices, + self.num_active, num_reqs, MAX_NUM_REQS=self.max_num_reqs, BLOCK_SIZE=triton.next_power_of_2(self.max_num_reqs), From 3842f54e0a3a6a142718e56efb7ecf0f32ce38a7 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Thu, 5 Nov 2026 13:24:34 +0100 Subject: [PATCH 02/14] [Mamba] Fix GDN ReplaySSM graph dispatch and simplify validation Clear stale graph indices and exclude intermediate GDN prefill from full graph dispatch. Reuse safe uniform MTP inputs and skip prefix materialization when exact host positions rule out a boundary. Separate GDN configuration checks, simplify selected FlashInfer API loading, and clarify the provider and cache contracts. Upgrade FlashInfer to 0.7.3. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- .buildkite/test_areas/engine.yaml | 8 +- docs/features/speculative_decoding/mtp.md | 21 ++- requirements/cuda.txt | 2 +- tests/kernels/mamba/test_gdn_replayssm.py | 59 +++++++- tests/kernels/mamba/test_ssu_dispatch.py | 24 ++-- tests/test_envs.py | 10 +- .../v1/attention/test_gdn_metadata_builder.py | 34 +++++ tests/v1/worker/test_gpu_batch_ordering.py | 15 ++ tests/v1/worker/test_gpu_model_runner.py | 65 +++++++++ .../worker/test_mamba_hybrid_model_state.py | 1 + vllm/config/vllm.py | 24 +++- vllm/envs.py | 17 ++- vllm/model_executor/layers/mamba/gdn/base.py | 6 +- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 16 ++- .../layers/mamba/gdn/replayssm.py | 128 ++++++++---------- .../layers/mamba/mamba_utils.py | 7 +- vllm/model_executor/models/qwen3_5.py | 2 + vllm/v1/attention/backends/gdn_attn.py | 4 + .../worker/gpu/model_states/mamba_hybrid.py | 22 ++- vllm/v1/worker/gpu_model_runner.py | 30 +++- vllm/v1/worker/mamba_utils.py | 10 ++ vllm/v1/worker/replayssm_utils.py | 11 +- 22 files changed, 398 insertions(+), 118 deletions(-) diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 5dc4ce2507eb..3a844b51631d 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -213,6 +213,7 @@ steps: source_file_dependencies: - tests/v1/e2e/test_replayssm_decode.py - tests/kernels/mamba/test_ssu_dispatch.py + - tests/kernels/mamba/test_gdn_replayssm.py - vllm/v1/core/sched/scheduler.py - vllm/v1/worker/utils.py - vllm/v1/kv_cache_interface.py @@ -232,7 +233,12 @@ steps: - vllm/v1/worker/gpu_worker.py commands: - VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v -s v1/e2e/test_replayssm_decode.py - - VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v kernels/mamba/test_ssu_dispatch.py::test_replayssm_materializes_only_accepted_boundary_with_compacted_rows kernels/mamba/test_ssu_dispatch.py::test_replayssm_postprocess_commits_staged_transition + - >- + VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v -s + kernels/mamba/test_ssu_dispatch.py::test_replayssm_materializes_only_accepted_boundary_with_compacted_rows + kernels/mamba/test_ssu_dispatch.py::test_replayssm_postprocess_commits_staged_transition + kernels/mamba/test_ssu_dispatch.py::test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers + kernels/mamba/test_gdn_replayssm.py mirror: amd: automatic_shard: true diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index 5a75d28ad31a..fe0c2a3991f0 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -82,17 +82,24 @@ vllm serve Qwen/Qwen3.5-0.8B \ --speculative-config '{"method":"mtp","num_speculative_tokens":3}' ``` -This backend supports one to seven speculative tokens. It requires BF16 state and +This backend supports standard decoding and one to seven speculative tokens. +Omit `--speculative-config` for standard decoding. It requires BF16 state and 128-element key and value heads. Pipeline parallelism, microbatching, and KV -connectors are unsupported. Adaptive verification is unsupported because GDN -output indices require fixed verification lengths. +connectors are unsupported. Ragged scheduled rows are supported. Adaptive +verification is unsupported because it changes row lengths on the GPU after the +CPU metadata builder determines the output indices. Set `enable_adaptive_verification` to `false` in the speculative configuration. Prefix caching requires `--mamba-cache-mode align`. -FlashInfer must provide `gated_delta_rule_stp_ucache_flush` and -`gated_delta_rule_mtp_ucache_flush`. Prefix caching also requires -`gdn_prefix_materialize`. Check these APIs before use; a version number alone does -not establish availability. This integration does not install missing kernels. +Standard decoding requires `gated_delta_rule_stp_ucache_flush`. Speculative +decoding requires `gated_delta_rule_mtp_ucache_flush`, available in FlashInfer +0.7.2.post1. Prefix caching also requires `gdn_prefix_materialize`, available in +FlashInfer 0.7.3. The STP API is not included in either release, so standard +decoding requires a compatible provider build. This integration does not install +missing kernels or change FlashInfer. + +Set `SGLANG_GDN_WY_STRIDED_QKV=1` before process startup. FlashInfer reads this +setting when its kernel module is imported; setting it later is insufficient. ## Notes diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 12545f8047c7..8ef19a48516e 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -10,7 +10,7 @@ torchaudio==2.11.0 torchvision==0.29.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version torchcodec >= 0.14 PyNvVideoCodec==2.0.4 -flashinfer-python==0.7.2.post1 +flashinfer-python==0.7.3 apache-tvm-ffi==0.1.11 tilelang==0.1.12 # >=1.31.0 keeps execution plans alive for the CUDA graphs recorded from them diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 6da884c488f5..9c0df773b38a 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -1,18 +1,46 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import builtins import math +from types import ModuleType import pytest import torch import torch.nn.functional as F from vllm.model_executor.layers.mamba.gdn.replayssm import ( + check_gdn_replayssm_dependencies, pack_replayssm_rows, run_gdn_replayssm, ) +@pytest.mark.parametrize("width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp")]) +def test_dependencies_import_only_selected_decode_api(monkeypatch, width, mode): + """An MTP-only FlashInfer installation must not need the STP module.""" + module = ModuleType("provider") + kernel = lambda **kwargs: None + setattr(module, f"gated_delta_rule_{mode}_ucache_flush", kernel) + suffix = "stp" if mode == "stp" else "flush" + expected = f"flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_{suffix}" + original_import = builtins.__import__ + + def import_provider(name, *args, **kwargs): + if name.startswith("flashinfer"): + assert name == expected, f"Unselected dependency: {name}" + return module + return original_import(name, *args, **kwargs) + + monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") + monkeypatch.setattr(builtins, "__import__", import_provider) + check_gdn_replayssm_dependencies.cache_clear() + try: + assert check_gdn_replayssm_dependencies(width, False) is kernel + finally: + check_gdn_replayssm_dependencies.cache_clear() + + def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): values = torch.tensor([[1.0], [2.0], [3.0]]) packed = pack_replayssm_rows( @@ -29,6 +57,31 @@ def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): ) +@pytest.mark.parametrize("width", [4, 8]) +def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): + values = torch.arange(2 * width * 4, dtype=torch.bfloat16).view(2 * width, 4) + starts = torch.tensor([0, width, 2 * width]) + packed = pack_replayssm_rows(values, starts, width, uniform=True) + assert packed.data_ptr() == values.data_ptr() + torch.testing.assert_close(packed, pack_replayssm_rows(values, starts, width)) + # A gate view can be contiguous but start at an unaligned byte offset. + unaligned = torch.zeros(values.numel() + 1, dtype=values.dtype)[1:].view_as(values) + unaligned.copy_(values) + aligned = pack_replayssm_rows(unaligned, starts, width, uniform=True) + assert aligned.data_ptr() % 16 == 0 + torch.testing.assert_close(aligned, packed) + strided = torch.zeros(2 * width, 8, dtype=values.dtype)[:, ::2] + strided.copy_(values) + torch.testing.assert_close( + pack_replayssm_rows(strided, starts, width, uniform=True), packed + ) + # Eager staging can include a zero-token graph-padding row. + padded_starts = torch.tensor([0, width, width]) + padded = pack_replayssm_rows(values[:width], padded_starts, width, uniform=True) + torch.testing.assert_close(padded[0], values[:width]) + assert padded[1].eq(0).all() + + def _reference_gdn_step( state, q, @@ -93,7 +146,10 @@ def _reconstruct_replayssm_state( not torch.cuda.is_available() or torch.cuda.get_device_capability() < (9, 0), reason="FlashInfer GDN ReplaySSM requires an SM90+ CUDA device", ) -@pytest.mark.parametrize("width,query_len,accepted", [(1, 1, 1), (4, 3, 2), (8, 6, 4)]) +@pytest.mark.parametrize( + "width,query_len,accepted", + [(1, 1, 1), (4, 3, 2), (4, 4, 2), (8, 6, 4), (8, 8, 4)], +) def test_gdn_replayssm_output_and_accepted_state( monkeypatch, width, query_len, accepted ): @@ -156,6 +212,7 @@ def random_tensor(*shape): state_indices=torch.tensor([1], dtype=torch.int32, device=device), output_indices=torch.arange(query_len, dtype=torch.int32, device=device), executed_query_width=width, + uniform_query_length=query_len, num_k_heads=1, num_v_heads=2, head_k_dim=128, diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index fff23106e3ea..e388257baee0 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -316,7 +316,10 @@ def test_replayssm_materializes_only_accepted_boundary_with_compacted_rows( @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("post_step", [False, True], ids=["v1", "v2"]) -def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_step): +@pytest.mark.parametrize("gdn_stp", [False, True]) +def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( + post_step, gdn_stp +): """Skip prefill self-copies, reset trackers, and preserve in-place MTP replay.""" mixer = _replayssm_mixer( kv_cache=( @@ -327,7 +330,7 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st torch.empty(6, 1, 20, 1, device="cuda"), ), ring_start=_cuda_ints([3] * 6), - committed=_cuda_ints([2] * 6), + committed=_cuda_ints([2, 2, 2, 2, 15 if gdn_stp else 2, 2]), A=torch.empty(1, device="cuda"), ) spec = MambaSpec( @@ -347,11 +350,14 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st context = ReplaySSMModelContext.create([([mixer], spec, table)], max_num_reqs=4) # Reordered rows: canonical prefill in slot 2; in-place MTP replay in slot 4; # padded slot; padded request. + query_len = 1 if gdn_stp else 4 _postprocess_replayssm_kernel[(4, 1)]( _cuda_ints([1, 2, 0, -1]), - _cuda_ints([0, 48, 52, 68, 68]) if post_step else _cuda_ints([48, 4, 16, 0]), - _cuda_ints([16, 48, 32]) if post_step else _cuda_ints([0, 0, 28]), - _cuda_ints([1, 1, 4]), + _cuda_ints([0, 48, 48 + query_len, 64 + query_len, 64 + query_len]) + if post_step + else _cuda_ints([48, query_len, 16, 0]), + _cuda_ints([16, 48, 32]) if post_step else _cuda_ints([0, 0, 32 - query_len]), + _cuda_ints([1, 1, query_len]), torch.tensor([True, False, True, True], device="cuda"), _cuda_ints([0, 2, 1]), context.block_table_ptrs, @@ -366,15 +372,15 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st MAX_NUM_REQS=4, MAMBA_BLOCK_SIZE=16, LOGICAL_WINDOW=16, - RING_BUFFER_LEN=20, + RING_BUFFER_LEN=16 if gdn_stp else 20, PAD_SLOT_ID=NULL_BLOCK_ID, QUERY_METADATA_IS_CUMULATIVE=post_step, NUM_COMPUTED_IS_POST_STEP=post_step, HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, - EXECUTED_QUERY_WIDTH=0, - GDN_STP=False, + EXECUTED_QUERY_WIDTH=1 if gdn_stp else 0, + GDN_STP=gdn_stp, ) _compact_replayssm_requests_kernel[(1,)]( context.plan_flush_count, @@ -385,7 +391,7 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st BLOCK_SIZE=4, ) - assert context.plan_flush_count.tolist() == [-1, 6, -1, -1] + assert context.plan_flush_count.tolist() == [-1, 0 if gdn_stp else 6, -1, -1] assert context.active_request_indices.tolist() == [1, -1, -1, -1] assert context.src_slots[0, 1].item() == 4 assert context.dst_slots[0, 1].item() == 4 diff --git a/tests/test_envs.py b/tests/test_envs.py index 2f3d0dc2f689..948551a9a5f8 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -300,9 +300,15 @@ def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False) assert env_func() == "cuda" - for value in ("cuda", "triton", "flashinfer_replayssm"): + for value in ( + "cuda", + "triton", + "flashinfer_replayssm", + "CUDA", + "FLASHINFER_REPLAYSSM", + ): monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) - assert env_func() == value + assert env_func() == value.lower() for value in ("fused", "invalid"): monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index e078b366c5a2..ae49cd60eb0b 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -188,6 +188,11 @@ def _build( ) -> GDNAttentionMetadata: """Build GDN attention metadata, optionally with spec-decode kwargs.""" common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE) + if builder.use_gdn_replayssm: + common = common.replace( + is_prefilling=torch.zeros(batch_spec.batch_size, dtype=torch.bool), + is_last_prefill=torch.zeros(batch_spec.batch_size, dtype=torch.bool), + ) if block_table is not None: common = common.replace(block_table_tensor=block_table) kwargs: dict = {} @@ -280,6 +285,35 @@ def test_has_initial_state_after_reclassification(): assert meta.has_initial_state[0].item() is True +def test_replayssm_captured_indices_remain_valid_after_ragged_batch(monkeypatch): + """A smaller FULL graph must not read stale indices from a ragged step.""" + config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE) + config.cache_config.use_replayssm = True + config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE + config.speculative_config = SpeculativeConfig( + method="ngram", num_speculative_tokens=3 + ) + monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", "flashinfer_replayssm") + builder = GDNAttentionMetadataBuilder( + kv_cache_spec=MambaSpec( + block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.bfloat16,) + ), + layer_names=["layer.0"], + vllm_config=config, + device=DEVICE, + ) + captured = _build(builder, BatchSpec(seq_lens=[80] * 32, query_lens=[4] * 32)) + indices = captured.replayssm_output_indices + assert indices is not None + _build(builder, BatchSpec(seq_lens=[80] * 33, query_lens=[1] + [4] * 32)) + _build(builder, BatchSpec(seq_lens=[80] * 32, query_lens=[4] * 30 + [0] * 2)) + # The graph retains its captured 128-entry view, not the new shorter view. + output = torch.arange(128) + selected = output.index_select(0, indices.long()) + torch.testing.assert_close(selected[:120], output[:120]) + assert selected[120:].eq(0).all() + + def test_full_cudagraph_spec_metadata_uses_request_count(): """FULL cudagraph token padding must not pad request-indexed metadata.""" num_speculative_tokens = 3 diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 51db644ab33d..ceb22ded1b4d 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -331,3 +331,18 @@ def test_no_speculator_dispatches_on_query_length_alone(): "these speculators classify a decode batch by query length alone; use " f"get_uniform_decode_token_count instead: {offenders}" ) + + +def test_replayssm_materialization_bound_covers_partial_acceptance(): + from vllm.v1.worker.replayssm_utils import may_cross_mamba_block_boundary + + for computed in (0, 12, 15, 16, 31): + for scheduled in (1, 4, 8): + may_cross = may_cross_mamba_block_boundary( + np.array([computed]), np.array([scheduled]), 16 + ) + for accepted in range(1, scheduled + 1): + crosses = computed // 16 != (computed + accepted) // 16 + assert may_cross or not crosses + assert not may_cross_mamba_block_boundary(np.array([16]), np.array([8]), 16) + assert may_cross_mamba_block_boundary(np.array([15]), np.array([1]), 16) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index ecb05e72507f..f13d1980496b 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -2026,3 +2026,68 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks(): with pytest.raises(ValueError, match="max_num_seqs"): runner.initialize_kv_cache(kv_cache_config) + + +def test_gdn_intermediate_prefill_rejects_full_graph_dispatch(dist_init, monkeypatch): + from vllm.config import CompilationConfig, CUDAGraphMode + from vllm.v1.worker.gpu_input_batch import CachedRequestState + + monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", "flashinfer_replayssm") + monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") + config = VllmConfig( + model_config=ModelConfig(model="Qwen/Qwen3.5-0.8B", dtype="bfloat16"), + scheduler_config=SchedulerConfig( + max_num_seqs=1, + max_num_batched_tokens=128, + max_model_len=128, + is_encoder_decoder=False, + ), + cache_config=CacheConfig( + use_replayssm=True, + replayssm_buffer_len=16, + mamba_cache_dtype="bfloat16", + mamba_ssm_cache_dtype="bfloat16", + mamba_cache_mode="none", + enable_prefix_caching=False, + ), + compilation_config=CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_DECODE_ONLY, + cudagraph_capture_sizes=[1], + max_cudagraph_capture_size=1, + ), + ) + with set_current_vllm_config(config): + runner = GPUModelRunner(config, torch.device(DEVICE_TYPE)) + runner.cudagraph_dispatcher.initialize_cudagraph_keys( + CUDAGraphMode.FULL_DECODE_ONLY, uniform_decode_query_len=1 + ) + # A one-token intermediate chunk has a decode shape but needs prefill. + # Final prompt tails with prior state and ordinary decode retain FULL. + for computed, expected in ( + (64, CUDAGraphMode.NONE), + (67, CUDAGraphMode.FULL), + (68, CUDAGraphMode.FULL), + ): + runner.input_batch.add_request( + CachedRequestState( + req_id="gdn", + prompt_token_ids=[1] * 68, + mm_features=[], + sampling_params=SamplingParams(temperature=0), + generator=None, + block_ids=tuple( + [] for _ in runner.input_batch.block_table.block_tables + ), + num_computed_tokens=computed, + output_token_ids=[], + ) + ) + runner.input_batch.refresh_metadata() + mode, *_ = runner._determine_batch_execution_and_padding( + num_tokens=1, + num_reqs=1, + num_scheduled_tokens_np=np.array([1]), + max_num_scheduled_tokens=1, + ) + assert mode == expected + runner.input_batch.remove_request("gdn") diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index 8b0fbbacec28..50e61ac15e97 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -520,6 +520,7 @@ def test_flashinfer_replayssm_prefix_uses_original_accepted_counts() -> None: state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32, device="cuda") state._mamba_state_idx_gpu = torch.zeros(4, dtype=torch.int32, device="cuda") state._is_prefilling_gpu = torch.zeros(4, dtype=torch.bool, device="cuda") + state._replayssm_may_materialize = True state._replayssm_query_start_loc = torch.tensor( [0, 4], dtype=torch.int32, device="cuda" ) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index da35882e6917..6d868ce80635 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -129,8 +129,8 @@ ) # SupportsReplaySSM advertises support for some ReplaySSM implementation. This -# allowlist selects architectures validated for the FlashInfer GDN integration: -# sharing a layer or inheriting the marker does not validate the model lifecycle. +# allowlist defines the current FlashInfer GDN integration scope. Sharing a +# layer or inheriting the marker does not opt another model into this integration. _GDN_REPLAYSSM_ARCHITECTURES = frozenset( { "Qwen3_5ForCausalLM", @@ -3945,7 +3945,7 @@ def validate_mamba_block_size(self) -> "VllmConfig": return self @model_validator(mode="after") - def validate_mamba_cached_kernel(self) -> "VllmConfig": + def validate_gdn_replayssm(self) -> "VllmConfig": # Call the implementation on the class so this validator remains usable # by focused tests that pass a config-shaped object rather than a full # pydantic model. @@ -3988,6 +3988,7 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": gdn_replayssm_geometry, ) + # Integration constraints: fixed verify widths and none/align caches. if ( self.speculative_config is not None and self.speculative_config.enable_adaptive_verification @@ -4009,11 +4010,13 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": "FlashInfer GDN ReplaySSM prefix caching requires " "--mamba-cache-mode=align" ) + # Selected provider constraints: history tile and launch widths. if self.cache_config.replayssm_buffer_len != 16: raise ValueError( "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" ) gdn_replayssm_geometry(self.num_speculative_tokens) + # This integration selects the provider's BF16 path. if self.model_config is not None: if self.model_config.dtype != torch.bfloat16: raise ValueError( @@ -4036,6 +4039,7 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": raise ValueError( "FlashInfer GDN ReplaySSM requires a bfloat16 recurrent state" ) + # These execution modes need additional tracker lifecycle support. if self.parallel_config.pipeline_parallel_size > 1: raise ValueError( "FlashInfer GDN ReplaySSM currently requires " @@ -4046,11 +4050,13 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": "FlashInfer GDN ReplaySSM does not support overlapping " "microbatch execution" ) + # The provider selects its strided-QKV specialization at import time. if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": raise ValueError( "FlashInfer GDN ReplaySSM requires " "SGLANG_GDN_WY_STRIDED_QKV=1 before process startup" ) + # This integration does not transfer replay rings and trackers. if ( self.kv_transfer_config is not None and self.kv_transfer_config.is_kv_transfer_instance @@ -4060,8 +4066,16 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": "(P/D disaggregation, KV cache offload)" ) self.cache_config.use_kda_recoverssm = False - # The GDN allowlist above validates this integration's lifecycle; - # the generic SupportsReplaySSM marker alone is not sufficient. + return self + return self + + @model_validator(mode="after") + def validate_mamba_cached_kernel(self) -> "VllmConfig": + if not self.cache_config.use_replayssm: + self.cache_config.use_kda_recoverssm = False + return self + # The preceding GDN validator owns its provider and lifecycle checks. + if VllmConfig.is_gdn_replayssm_enabled(self): return self kda_architectures = ( diff --git a/vllm/envs.py b/vllm/envs.py index 674094968a54..426cb4baf5db 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -9,7 +9,7 @@ import tempfile import uuid from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, cast if TYPE_CHECKING: VLLM_HOST_IP: str = "" @@ -1285,12 +1285,15 @@ def _resolve_rust_cli_path() -> str | None: # where supported and falls back to "triton" otherwise; explicitly selecting # "cuda" raises when unsupported. "flashinfer_replayssm" selects checkpoint # and replay execution and requires --use-replayssm. - "VLLM_GDN_DECODE_KERNEL": env_with_choices( - "VLLM_GDN_DECODE_KERNEL", - "cuda", - ["cuda", "triton", "flashinfer_replayssm"], - case_sensitive=False, - ), + "VLLM_GDN_DECODE_KERNEL": lambda: cast( + str, + env_with_choices( + "VLLM_GDN_DECODE_KERNEL", + "cuda", + ["cuda", "triton", "flashinfer_replayssm"], + case_sensitive=False, + )(), + ).lower(), # Disable pynccl (using torch.distributed instead) "VLLM_DISABLE_PYNCCL": lambda: ( os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1") diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index b453a66f3472..3329dcc0b854 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -15,6 +15,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, gdn_replayssm_geometry, + mamba_state_alignment, ) from vllm.model_executor.models.utils import extract_layer_index from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -48,7 +49,7 @@ def __init__( ) self.use_replayssm = vllm_config.is_gdn_replayssm_enabled() self.use_flashinfer_replayssm = self.use_replayssm - self.state_alignment = 16 if self.use_flashinfer_replayssm else 1 + self.state_alignment = mamba_state_alignment(vllm_config) self.replayssm_buffer_len = ( self.cache_config.replayssm_buffer_len if self.use_replayssm else None ) @@ -59,7 +60,8 @@ def __init__( gdn_replayssm_geometry(self.num_spec) ) if self.replayssm_executed_query_width is not None: - # STP executes one token but still uses four staging rows. + # STP accepts four input positions: one real token and three zeros. + # These offsets pack token inputs; they are not replay-ring cursors. packed_width = max(self.replayssm_executed_query_width, 4) self.register_buffer( "_replayssm_offsets", diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 949032e95058..357bc483445c 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -925,20 +925,23 @@ def _flashinfer_replayssm_unsupported_reason( ) -> str | None: conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2] if self.gqa_interleaved_layout: - return "Qwen3-Next's interleaved GQA layout has not been validated" + return "interleaved GQA layout is not supported by this integration" if self.head_k_dim != 128 or self.head_v_dim != 128: - return "the FlashInfer kernel requires K=V=128" + return "the selected FlashInfer kernel requires K=V=128" if self.num_k_heads % self.tp_size or self.num_v_heads % self.tp_size: return "tensor-parallel head counts must divide evenly" if self.num_v_heads % self.num_k_heads: return "value heads must be an integer multiple of key heads" if vllm_config.model_config.dtype != torch.bfloat16: - return "model activations must use bfloat16" + return "the GDN ReplaySSM adapter requires bfloat16 model activations" if ( conv_state_dtype != torch.bfloat16 or recurrent_state_dtype != torch.bfloat16 ): - return "convolution and recurrent state must use bfloat16" + return ( + "the GDN ReplaySSM adapter requires bfloat16 convolution " + "and recurrent state" + ) if not current_platform.has_device_capability(90): return "the initial integration requires an SM90+ CUDA device" return None @@ -1958,6 +1961,8 @@ def _forward_core( mixed_qkv_non_spec = None if self.use_replayssm and spec_sequence_masks is not None: + # ReplaySSM splits packed QKV inside its adapter. Other decode + # paths consume the separate query, key, and value views below. query_spec, key_spec, value_spec = None, None, None else: query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) @@ -2023,7 +2028,7 @@ def _forward_core( # 2. Recurrent attention # 2.1: Process the multi-query part - if spec_sequence_masks is not None and self.use_replayssm: + if self.use_replayssm and spec_sequence_masks is not None: assert mixed_qkv_spec is not None assert a_spec is not None and b_spec is not None assert spec_state_indices_tensor is not None @@ -2044,6 +2049,7 @@ def _forward_core( query_start_loc=spec_query_start_loc[: num_replay_rows + 1], state_indices=spec_state_indices_tensor[:num_replay_rows, 0], output_indices=attn_metadata.replayssm_output_indices, + uniform_query_length=attn_metadata.uniform_spec_sequence_length, executed_query_width=(attn_metadata.replayssm_executed_query_width), num_k_heads=self.num_k_heads // self.tp_size, num_v_heads=self.num_v_heads // self.tp_size, diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index 54fe0d29c039..d6009277c498 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -14,62 +14,35 @@ @cache -def _check_gdn_replayssm_environment() -> None: +def check_gdn_replayssm_dependencies( + executed_width: int, needs_materializer: bool +) -> Callable[..., torch.Tensor]: + """Validate the environment and return only the selected decode API.""" + if executed_width not in (1, 4, 8): + raise ValueError(f"Unsupported GDN ReplaySSM executed width: {executed_width}") if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": raise RuntimeError( "FlashInfer GDN ReplaySSM requires " "SGLANG_GDN_WY_STRIDED_QKV=1 before importing FlashInfer" ) - - -@cache -def _load_gdn_replayssm_mtp_kernel() -> Callable[..., torch.Tensor]: - _check_gdn_replayssm_environment() - try: - from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_flush import ( - gated_delta_rule_mtp_ucache_flush, - ) - except ImportError as e: - raise ImportError( - "FlashInfer GDN ReplaySSM requires a compatible flashinfer-python " - "build exposing gated_delta_rule_mtp_ucache_flush" - ) from e - return gated_delta_rule_mtp_ucache_flush - - -@cache -def _load_gdn_replayssm_stp_kernel() -> Callable[..., torch.Tensor]: - _check_gdn_replayssm_environment() - try: - from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_stp import ( - gated_delta_rule_stp_ucache_flush, - ) - except ImportError as e: - raise ImportError( - "FlashInfer GDN ReplaySSM STP requires a compatible " - "flashinfer-python build exposing " - "gated_delta_rule_stp_ucache_flush" - ) from e - return gated_delta_rule_stp_ucache_flush - - -def check_gdn_replayssm_dependencies( - executed_width: int, needs_materializer: bool -) -> None: - """Check selected FlashInfer exports during GPU worker layer construction.""" - if executed_width == 1: - loader = _load_gdn_replayssm_stp_kernel - api = "gated_delta_rule_stp_ucache_flush" - elif executed_width in (4, 8): - loader = _load_gdn_replayssm_mtp_kernel - api = "gated_delta_rule_mtp_ucache_flush" - else: - raise ValueError(f"Unsupported GDN ReplaySSM executed width: {executed_width}") + api = ( + "gated_delta_rule_stp_ucache_flush" + if executed_width == 1 + else "gated_delta_rule_mtp_ucache_flush" + ) mode = f"executed width={executed_width}, cache mode=" + ( "align" if needs_materializer else "none" ) try: - loader() + # MTP-only providers need not ship the separate STP API. + if executed_width == 1: + from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_stp import ( + gated_delta_rule_stp_ucache_flush as kernel, + ) + else: + from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_flush import ( + gated_delta_rule_mtp_ucache_flush as kernel, + ) except ImportError as e: raise ImportError( f"FlashInfer GDN ReplaySSM ({mode}) requires {api}: {e}" @@ -86,6 +59,7 @@ def check_gdn_replayssm_dependencies( f"FlashInfer GDN ReplaySSM ({mode}) requires " f"gdn_prefix_materialize: {e}" ) from e + return kernel def pack_replayssm_rows( @@ -94,9 +68,17 @@ def pack_replayssm_rows( executed_query_width: int, offsets: torch.Tensor | None = None, padding_value: float = 0.0, + uniform: bool = False, ) -> torch.Tensor: """Pack flattened ragged rows into fixed-width rows.""" num_rows = query_start_loc.numel() - 1 + if ( + uniform + and values.shape[0] == num_rows * executed_query_width + and values.is_contiguous() + and values.data_ptr() % 16 == 0 + ): + return values.view(num_rows, executed_query_width, *values.shape[1:]) if num_rows == 0: return values.new_empty((0, executed_query_width, *values.shape[1:])) if values.shape[0] == 0: @@ -142,12 +124,13 @@ def run_gdn_replayssm( head_v_dim: int, offsets: torch.Tensor | None = None, kernel: Callable[..., torch.Tensor] | None = None, + uniform_query_length: int | None = None, ) -> torch.Tensor: """Pack rows, invoke FlashInfer, and return only real-token outputs. STP receives one real row per request. The adapter supplies FlashInfer's - caller-owned padded T=4 staging layout, while vLLM keeps cursor publication - model-wide so every layer observes the same pre-step cursor. + four-token input buffer: one real token followed by three zero rows. + vLLM publishes cursors once per model step, after every layer runs. """ if executed_query_width not in (1, 4, 8): raise ValueError( @@ -167,7 +150,10 @@ def run_gdn_replayssm( # caller-owned zero-padded T=4 buffer. Packing a shared QKV allocation here # preserves the common token stride required by block-strided cache pools. packed_width = 4 if executed_query_width == 1 else executed_query_width - packed_qkv = pack_replayssm_rows(mixed_qkv, query_start_loc, packed_width, offsets) + uniform = executed_query_width > 1 and uniform_query_length == packed_width + packed_qkv = pack_replayssm_rows( + mixed_qkv, query_start_loc, packed_width, offsets, uniform=uniform + ) # CuTe requires every input pointer to be 16-byte aligned. Packing the two # gates together and splitting them can leave ``b`` at a small byte offset # into the shared allocation when the local head count is not a multiple of @@ -176,15 +162,19 @@ def run_gdn_replayssm( # recurrent no-ops: a=-inf gives zero decay after softplus(a + dt_bias), # while zero q/k/v and b give a zero state update. STP has no ragged real # rows, so only invalid graph-padding requests can be padded here; their - # negative state index skips the kernel CTA. + # negative state index skips the kernel CTA. STP requires three zero + # input rows after the real token. It returns and commits only that token. packed_a = pack_replayssm_rows( a, query_start_loc, packed_width, offsets, padding_value=0.0 if executed_query_width == 1 else -torch.inf, + uniform=uniform, + ) + packed_b = pack_replayssm_rows( + b, query_start_loc, packed_width, offsets, uniform=uniform ) - packed_b = pack_replayssm_rows(b, query_start_loc, packed_width, offsets) local_k_dim = num_k_heads * head_k_dim local_v_dim = num_v_heads * head_v_dim q_flat, k_flat, v_flat = packed_qkv.split( @@ -212,12 +202,16 @@ def run_gdn_replayssm( ) if kernel is None: - kernel = ( - _load_gdn_replayssm_stp_kernel() - if executed_query_width == 1 - else _load_gdn_replayssm_mtp_kernel() - ) - common_kwargs = dict( + kernel = check_gdn_replayssm_dependencies(executed_query_width, False) + if executed_query_width == 1: + # STP updates these cursor copies. Every layer must start from the + # same pre-step values; model-wide postprocess publishes the update. + mode_kwargs = {"prepadded": True} + flush_min = GDN_REPLAY_LOGICAL_WINDOW - 1 + else: + mode_kwargs = {"restart_hist_on_flush": False} + flush_min = GDN_REPLAY_LOGICAL_WINDOW + 1 - executed_query_width + kernel( # These are immutable model parameters. CuTe's DLPack bridge rejects # tensors whose Parameter wrapper still advertises autograd tracking, # even while vLLM executes under inference mode. @@ -238,23 +232,9 @@ def run_gdn_replayssm( g_cache=g_cache, hist_len=hist_len, cache_base=cache_base, + flush_min=flush_min, + **mode_kwargs, ) - if executed_query_width == 1: - # The STP wrapper self-commits these gathered cursor copies. Do not - # scatter them into the pool here: subsequent GDN layers must consume - # the same pre-step values. The model-wide postprocess publishes the - # equivalent transition once after the target step completes. - kernel( - **common_kwargs, - flush_min=GDN_REPLAY_LOGICAL_WINDOW - 1, - prepadded=True, - ) - else: - kernel( - **common_kwargs, - flush_min=GDN_REPLAY_LOGICAL_WINDOW + 1 - executed_query_width, - restart_hist_on_flush=False, - ) return output.flatten(0, 1).index_select(0, output_indices.long()) diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index f5ab1387be57..d39dc156a46b 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -29,7 +29,12 @@ def gdn_replayssm_geometry(num_speculative_tokens: int) -> tuple[int, int]: - """Return GDN's executed query width and physical replay ring slots.""" + """Return the selected GDN provider's query width and physical ring slots. + + Both paths retain at most 16 logical history records. MTP needs extra + physical slots for verification writes while other blocks read history. + STP appends one record or folds the current token without appending. + """ if num_speculative_tokens == 0: return 1, 16 if 1 <= num_speculative_tokens <= 3: diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 7760c3ff3801..695589e97480 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -634,6 +634,8 @@ def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", ) -> tuple[torch.dtype, ...]: + # The language model owns the recurrent layers. Reuse its cache schema, + # including ReplaySSM rings, for this multimodal wrapper. return Qwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config(vllm_config) @classmethod diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 0cfbf2bf1ba6..1bfc5a570042 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -229,6 +229,8 @@ def __init__( self.compilation_config.max_cudagraph_capture_size, ) + # ReplaySSM uses one canonical state plus replay records. The ordinary + # speculative path keeps a separate state index for each token. self.state_table_width = 1 if self.use_gdn_replayssm else self.num_spec + 1 self.spec_state_indices_tensor: torch.Tensor = torch.empty( (self.decode_cudagraph_max_bs, self.state_table_width), @@ -759,6 +761,8 @@ def build( # type: ignore[override] self.replayssm_output_indices[:output_count].copy_( replayssm_output_indices, non_blocking=True ) + # Captured graphs retain a longer view when live rows shrink. + self.replayssm_output_indices[output_count:].zero_() replayssm_output_indices = self.replayssm_output_indices[:output_count] if self._stage_decode(num_prefills, num_decodes, num_spec_decodes): diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index af708717e94c..2945bb9ac0b8 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -41,7 +41,10 @@ preprocess_mamba_align_fused_kernel, validate_mamba_state_copy_funcs, ) -from vllm.v1.worker.replayssm_utils import initialize_replayssm_ring_trackers +from vllm.v1.worker.replayssm_utils import ( + initialize_replayssm_ring_trackers, + may_cross_mamba_block_boundary, +) from vllm.v1.worker.utils import AttentionGroup @@ -99,6 +102,7 @@ def __init__( self.max_num_reqs, dtype=torch.int32, device=self.device ) self._replayssm_query_start_loc: torch.Tensor | None = None + self._replayssm_may_materialize = True # Pre-copy prefix-cache state (V2). The migration of each request's # mamba state across block boundaries runs as a fused GPU kernel reusing # the postprocess copy machinery, so the per-step src columns and the @@ -384,6 +388,21 @@ def prepare_attn( ) if self._use_flashinfer_replayssm: self._replayssm_query_start_loc = None + self._replayssm_may_materialize = True + if ( + self._use_gdn_replayssm + and self._align_mode + and self.vllm_config.num_speculative_tokens == 0 + ): + assert self._mamba_ctx is not None + replay = self._mamba_ctx.replayssm + assert replay is not None + # Speculative CPU positions are upper bounds, so keep MTP conservative. + self._replayssm_may_materialize = may_cross_mamba_block_boundary( + input_batch.num_computed_tokens_np, + input_batch.num_scheduled_tokens, + replay.mamba_block_size, + ) metadata_type = ( GDNAttentionMetadata if self._use_gdn_replayssm @@ -519,6 +538,7 @@ def _publish_flashinfer_replayssm( is_prefilling=self._is_prefilling_gpu[:num_reqs], live_cols=(self._mamba_state_idx_gpu if self._align_mode else None), num_reqs=num_reqs, + may_materialize=self._replayssm_may_materialize, ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 70599aed1787..460529cea462 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3719,6 +3719,20 @@ def _model_forward( **model_kwargs, ) + @staticmethod + def _gdn_has_prefill( + computed: np.ndarray, + prompt: np.ndarray, + query: np.ndarray, + draft: np.ndarray | None, + ) -> bool: + # Match GDNAttentionMetadataBuilder's canonical-prefill classification. + decode_rows = query == 1 + if draft is not None: + decode_rows |= (draft >= 0) & (query == draft + 1) + final_decode = (computed > 0) & (computed + query >= prompt) & decode_rows + return bool(((computed < prompt) & ~final_decode).any()) + @staticmethod def _is_uniform_decode( max_num_scheduled_tokens: int, @@ -3814,6 +3828,18 @@ def _determine_batch_execution_and_padding( num_reqs=num_reqs, force_uniform_decode=force_uniform_decode, ) + gdn_prefill = ( + force_uniform_decode is None + and self.vllm_config.is_gdn_replayssm_enabled() + and self._gdn_has_prefill( + self.input_batch.num_computed_tokens_cpu[:num_reqs], + self.input_batch.num_prompt_tokens[:num_reqs], + num_scheduled_tokens_np[:num_reqs], + self.num_decode_draft_tokens.np[:num_reqs] + if self.speculative_config is not None + else None, + ) + ) # Encoder-decoder models only support CG for decoder_step > 0 (no enc_output # is present). Also, chunked-prefill is disabled, so batch are uniform. has_encoder_output = ( @@ -3835,7 +3861,9 @@ def dispatch_cudagraph(num_tokens, disable_full=False, valid_modes=None): uniform_decode=uniform_decode, num_active_loras=num_active_loras, valid_modes={CUDAGraphMode.NONE} if force_eager else valid_modes, - invalid_modes={CUDAGraphMode.FULL} if disable_full else None, + invalid_modes={CUDAGraphMode.FULL} + if disable_full or gdn_prefill + else None, ) cudagraph_mode, batch_descriptor = dispatch_cudagraph( diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 9ab10f2b7052..e05794b41004 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -34,6 +34,7 @@ ReplaySSMModelContext, _reinterpret_u64_as_i64, mamba_state_copy_boundary, + may_cross_mamba_block_boundary, ) logger = init_logger(__name__) @@ -1759,6 +1760,15 @@ def postprocess_mamba_gpu( is_prefilling=ctx.is_prefilling_buf.gpu, live_cols=live_cols, num_reqs=num_reqs, + may_materialize=( + not ctx.replayssm.materialize_prefixes + or ctx.replayssm.mamba_type != MambaAttentionBackendEnum.GDN_ATTN + or may_cross_mamba_block_boundary( + ctx.num_computed_tokens_buf.np[:num_reqs], + ctx.num_scheduled_tokens_buf.np[:num_reqs], + ctx.replayssm.mamba_block_size, + ) + ), ) if num_accepted_tokens_cpu_tensor is not None and ctx.replayssm is None: diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index af015ffa2a8e..c8fbaf175263 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -7,6 +7,7 @@ from functools import cache from typing import Any, NamedTuple +import numpy as np import torch from vllm.model_executor.layers.mamba.abstract import MambaBase @@ -20,6 +21,13 @@ from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +def may_cross_mamba_block_boundary( + computed: np.ndarray, scheduled: np.ndarray, block_size: int +) -> bool: + """Bound accepted progress using exact pre-step counts and scheduled lengths.""" + return bool(((computed + scheduled) // block_size > computed // block_size).any()) + + def initialize_replayssm_ring_trackers( kv_cache_config: KVCacheConfig, forward_context: Mapping[str, Any], @@ -637,6 +645,7 @@ def postprocess( is_prefilling: torch.Tensor, live_cols: torch.Tensor | None, num_reqs: int, + may_materialize: bool = True, ) -> None: """Commit all groups before publishing their canonical prefix states.""" if num_reqs == 0: @@ -673,7 +682,7 @@ def postprocess( and self.executed_query_width == 1 ), ) - if not self.materialize_prefixes: + if not self.materialize_prefixes or not may_materialize: return _compact_replayssm_requests_kernel[(1,)]( self.plan_flush_count, From 79812bb3753d5a797a7f3258a2727838d55c5d1e Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Thu, 5 Nov 2026 14:07:14 +0100 Subject: [PATCH 03/14] [Mamba] Reduce GDN ReplaySSM input packing Copy one-token decode inputs directly into zero-filled buffers. Reuse aligned outer-strided QKV views for uniform MTP and keep gate layouts compact. Preserve ragged fallback, invalid-row handling, and per-layer cursor ownership. Cover captured STP padding and both full-width MTP specializations. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- tests/kernels/mamba/test_gdn_replayssm.py | 83 +++++++++++++++++-- .../layers/mamba/gdn/replayssm.py | 41 ++++++++- 2 files changed, 113 insertions(+), 11 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 9c0df773b38a..acb4982be9b1 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -72,9 +72,19 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): torch.testing.assert_close(aligned, packed) strided = torch.zeros(2 * width, 8, dtype=values.dtype)[:, ::2] strided.copy_(values) - torch.testing.assert_close( - pack_replayssm_rows(strided, starts, width, uniform=True), packed + repacked = pack_replayssm_rows( + strided, starts, width, uniform=True, allow_strided_rows=True + ) + assert repacked.is_contiguous() + torch.testing.assert_close(repacked, packed) + projection = torch.zeros(2 * width, 8, dtype=values.dtype)[:, :4] + projection.copy_(values) + viewed = pack_replayssm_rows( + projection, starts, width, uniform=True, allow_strided_rows=True ) + assert viewed.data_ptr() == projection.data_ptr() + assert viewed.stride() == (width * 8, 8, 1) + torch.testing.assert_close(viewed, packed) # Eager staging can include a zero-token graph-padding row. padded_starts = torch.tensor([0, width, width]) padded = pack_replayssm_rows(values[:width], padded_starts, width, uniform=True) @@ -82,6 +92,20 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): assert padded[1].eq(0).all() +@pytest.mark.parametrize("query_lens", [[1, 1], [1, 0]]) +def test_single_token_packing_matches_ragged_path(query_lens): + """STP copies strided inputs and preserves zero-token padding rows.""" + num_tokens = sum(query_lens) + values = torch.arange(num_tokens * 8, dtype=torch.bfloat16).view(num_tokens, 8) + values = values[:, ::2] + starts = torch.tensor([0, *torch.tensor(query_lens).cumsum(0).tolist()]) + packed = pack_replayssm_rows(values, starts, 4, single_token=True) + expected = pack_replayssm_rows(values, starts, 4) + torch.testing.assert_close(packed, expected, rtol=0, atol=0) + assert packed.data_ptr() % 16 == 0 + assert packed[:, 1:].eq(0).all() + + def _reference_gdn_step( state, q, @@ -184,20 +208,35 @@ def random_tensor(*shape): states.append(pages[:, offset : offset + size].view(dt).view(2, *shape)) offset += size checkpoint, u_cache, k_cache, g_cache = states + pages[0].fill_(17) # Nonzero canaries catch writes to the padding page. checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) ring_start = torch.zeros(2, dtype=torch.int32, device=device) committed = torch.zeros_like(ring_start) A_log = torch.full((2,), -3.0, device=device) dt_bias = random_tensor(2).float() * 0.2 accepted_state = checkpoint[1].float().clone() + padding_page = pages[0].clone() + graph = None # Eighteen steps cross the native STP flush and the MTP ring boundary. for _ in range(18): q, k = random_tensor(query_len, 1, 128), random_tensor(query_len, 1, 128) v = random_tensor(query_len, 2, 128) * 0.1 a, b = random_tensor(query_len, 2) * 0.2, random_tensor(query_len, 2) - output = run_gdn_replayssm( - mixed_qkv=torch.cat((q.flatten(1), k.flatten(1), v.flatten(1)), dim=-1), + mixed_qkv = torch.cat((q.flatten(1), k.flatten(1), v.flatten(1)), dim=-1) + if width == query_len and width in (4, 8): + projection = mixed_qkv.new_zeros(query_len, mixed_qkv.shape[1] + 128) + projection[:, : mixed_qkv.shape[1]].copy_(mixed_qkv) + mixed_qkv = projection[:, : mixed_qkv.shape[1]] + if width == 1: + # Captured padding has input storage but no token or live state. + mixed_qkv = torch.cat((mixed_qkv, torch.ones_like(mixed_qkv))) + a, b = ( + torch.cat((a, torch.ones_like(a))), + torch.cat((b, torch.ones_like(b))), + ) + inputs = dict( + mixed_qkv=mixed_qkv, a=a, b=b, A_log=A_log, @@ -207,10 +246,16 @@ def random_tensor(*shape): ring_start=ring_start, num_committed=committed, query_start_loc=torch.tensor( - [0, query_len], dtype=torch.int32, device=device + [0, 1, 1] if width == 1 else [0, query_len], + dtype=torch.int32, + device=device, + ), + state_indices=torch.tensor( + [1, 0] if width == 1 else [1], dtype=torch.int32, device=device + ), + output_indices=torch.arange( + 2 if width == 1 else query_len, dtype=torch.int32, device=device ), - state_indices=torch.tensor([1], dtype=torch.int32, device=device), - output_indices=torch.arange(query_len, dtype=torch.int32, device=device), executed_query_width=width, uniform_query_length=query_len, num_k_heads=1, @@ -218,6 +263,30 @@ def random_tensor(*shape): head_k_dim=128, head_v_dim=128, ) + if width == 1: + if graph is None: + graph_inputs = inputs + saved_pages = pages.clone() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run_gdn_replayssm(**graph_inputs) + torch.cuda.current_stream().wait_stream(stream) + pages.copy_(saved_pages) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_output = run_gdn_replayssm(**graph_inputs) + pages.copy_(saved_pages) + else: + for name in ("mixed_qkv", "a", "b"): + graph_inputs[name].copy_(inputs[name]) + graph.replay() + output = graph_output[:1] + assert graph_output[1].eq(0).all() + torch.testing.assert_close(pages[0], padding_page, rtol=0, atol=0) + a, b = a[:1], b[:1] + else: + output = run_gdn_replayssm(**inputs) expected_output, _ = _reference_gdn_step( accepted_state, q, k, v, a, b, A_log, dt_bias ) diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index d6009277c498..8693cba44dba 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -69,13 +69,28 @@ def pack_replayssm_rows( offsets: torch.Tensor | None = None, padding_value: float = 0.0, uniform: bool = False, + single_token: bool = False, + allow_strided_rows: bool = False, ) -> torch.Tensor: - """Pack flattened ragged rows into fixed-width rows.""" + """Pack ragged rows; single_token requires at most one token per row.""" num_rows = query_start_loc.numel() - 1 + if single_token and padding_value == 0.0 and values.shape[0] == num_rows: + packed = values.new_zeros((num_rows, executed_query_width, *values.shape[1:])) + packed[:, 0].copy_(values) + return packed if ( uniform and values.shape[0] == num_rows * executed_query_width - and values.is_contiguous() + and ( + values.is_contiguous() + or ( + allow_strided_rows + and values.ndim == 2 + and values.stride(1) == 1 + and values.stride(0) >= values.shape[1] + and values.stride(0) * values.element_size() % 16 == 0 + ) + ) and values.data_ptr() % 16 == 0 ): return values.view(num_rows, executed_query_width, *values.shape[1:]) @@ -151,8 +166,20 @@ def run_gdn_replayssm( # preserves the common token stride required by block-strided cache pools. packed_width = 4 if executed_query_width == 1 else executed_query_width uniform = executed_query_width > 1 and uniform_query_length == packed_width + # STP metadata permits only zero or one token per row. Eager zero-length + # rows use the packer's fallback. Captured padding can have input storage; + # its negative state index makes FlashInfer skip that row. + single_token = executed_query_width == 1 packed_qkv = pack_replayssm_rows( - mixed_qkv, query_start_loc, packed_width, offsets, uniform=uniform + mixed_qkv, + query_start_loc, + packed_width, + offsets, + uniform=uniform, + single_token=single_token, + # FlashInfer accepts Q/K/V views with a common aligned token stride. + # Gates keep their compact layout independently of this QKV path. + allow_strided_rows=True, ) # CuTe requires every input pointer to be 16-byte aligned. Packing the two # gates together and splitting them can leave ``b`` at a small byte offset @@ -171,9 +198,15 @@ def run_gdn_replayssm( offsets, padding_value=0.0 if executed_query_width == 1 else -torch.inf, uniform=uniform, + single_token=single_token, ) packed_b = pack_replayssm_rows( - b, query_start_loc, packed_width, offsets, uniform=uniform + b, + query_start_loc, + packed_width, + offsets, + uniform=uniform, + single_token=single_token, ) local_k_dim = num_k_heads * head_k_dim local_v_dim = num_v_heads * head_v_dim From 73546eeb83cb87f55aa9a45abeb2979f054c31bc Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Thu, 5 Nov 2026 15:40:01 +0100 Subject: [PATCH 04/14] [Mamba] Avoid ragged packing for uniform GDN inputs Copy uniform strided gates directly into aligned contiguous storage. Preserve views for supported QKV layouts and retain the ragged fallback. Keep fresh parameter conversions compatible with in-place weight reloads. Extend the existing CUDA recurrence case to cover updated weights after graph capture. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- tests/kernels/mamba/test_gdn_replayssm.py | 15 ++++++-- .../layers/mamba/gdn/replayssm.py | 34 +++++++++---------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index acb4982be9b1..b85898cf9ef0 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -85,6 +85,9 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): assert viewed.data_ptr() == projection.data_ptr() assert viewed.stride() == (width * 8, 8, 1) torch.testing.assert_close(viewed, packed) + gate = pack_replayssm_rows(projection, starts, width, uniform=True) + assert gate.is_contiguous() and gate.data_ptr() % 16 == 0 + torch.testing.assert_close(gate, packed) # Eager staging can include a zero-token graph-padding row. padded_starts = torch.tensor([0, width, width]) padded = pack_replayssm_rows(values[:width], padded_starts, width, uniform=True) @@ -174,6 +177,7 @@ def _reconstruct_replayssm_state( "width,query_len,accepted", [(1, 1, 1), (4, 3, 2), (4, 4, 2), (8, 6, 4), (8, 8, 4)], ) +@torch.inference_mode() def test_gdn_replayssm_output_and_accepted_state( monkeypatch, width, query_len, accepted ): @@ -212,14 +216,17 @@ def random_tensor(*shape): checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) ring_start = torch.zeros(2, dtype=torch.int32, device=device) committed = torch.zeros_like(ring_start) - A_log = torch.full((2,), -3.0, device=device) - dt_bias = random_tensor(2).float() * 0.2 + A_log = torch.nn.Parameter(torch.full((2,), -3.0, device=device)) + dt_bias = torch.nn.Parameter(random_tensor(2) * 0.2) accepted_state = checkpoint[1].float().clone() padding_page = pages[0].clone() graph = None # Eighteen steps cross the native STP flush and the MTP ring boundary. - for _ in range(18): + for step in range(18): + if width == 1 and step == 9: + # Captured execution must observe an in-place weight reload. + A_log.add_(2.0) q, k = random_tensor(query_len, 1, 128), random_tensor(query_len, 1, 128) v = random_tensor(query_len, 2, 128) * 0.1 a, b = random_tensor(query_len, 2) * 0.2, random_tensor(query_len, 2) @@ -228,6 +235,8 @@ def random_tensor(*shape): projection = mixed_qkv.new_zeros(query_len, mixed_qkv.shape[1] + 128) projection[:, : mixed_qkv.shape[1]].copy_(mixed_qkv) mixed_qkv = projection[:, : mixed_qkv.shape[1]] + gates = torch.stack((a, b), dim=1) + a, b = gates[:, 0], gates[:, 1] if width == 1: # Captured padding has input storage but no token or live state. mixed_qkv = torch.cat((mixed_qkv, torch.ones_like(mixed_qkv))) diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index 8693cba44dba..71e87bf4e93e 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -78,21 +78,21 @@ def pack_replayssm_rows( packed = values.new_zeros((num_rows, executed_query_width, *values.shape[1:])) packed[:, 0].copy_(values) return packed - if ( - uniform - and values.shape[0] == num_rows * executed_query_width - and ( - values.is_contiguous() - or ( - allow_strided_rows - and values.ndim == 2 - and values.stride(1) == 1 - and values.stride(0) >= values.shape[1] - and values.stride(0) * values.element_size() % 16 == 0 + if uniform and values.shape[0] == num_rows * executed_query_width: + if not ( + values.data_ptr() % 16 == 0 + and ( + values.is_contiguous() + or ( + allow_strided_rows + and values.ndim == 2 + and values.stride(1) == 1 + and values.stride(0) >= values.shape[1] + and values.stride(0) * values.element_size() % 16 == 0 + ) ) - ) - and values.data_ptr() % 16 == 0 - ): + ): + values = values.clone(memory_format=torch.contiguous_format) return values.view(num_rows, executed_query_width, *values.shape[1:]) if num_rows == 0: return values.new_empty((0, executed_query_width, *values.shape[1:])) @@ -245,9 +245,9 @@ def run_gdn_replayssm( mode_kwargs = {"restart_hist_on_flush": False} flush_min = GDN_REPLAY_LOGICAL_WINDOW + 1 - executed_query_width kernel( - # These are immutable model parameters. CuTe's DLPack bridge rejects - # tensors whose Parameter wrapper still advertises autograd tracking, - # even while vLLM executes under inference mode. + # CuTe rejects Parameters that advertise autograd tracking. Fresh + # detached wrappers also prevent stale FlashInfer casts after in-place + # weight reloads. Captured casts read the live parameter storage. A_log=A_log.detach(), a=packed_a, dt_bias=dt_bias.detach(), From 3f9c3c93d758782bba94b07bb8265338360f1fef Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Thu, 5 Nov 2026 15:54:48 +0100 Subject: [PATCH 05/14] [Mamba] Add FP16 GDN ReplaySSM through direct UMMA Use FlashInfer 0.7.3 direct UMMA for explicit FP16 recurrent state on SM100-family GPUs. Keep the existing page-strided cache layout and require speculative decoding without prefix caching or stochastic rounding. Preserve the BF16 path. Cover page-strided recurrence, captured padding, and in-place weight updates in the existing kernel cases. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/speculative_decoding/mtp.md | 12 ++- tests/kernels/mamba/test_gdn_replayssm.py | 85 +++++++++++++------ vllm/config/vllm.py | 21 ++++- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 20 +++-- .../layers/mamba/gdn/replayssm.py | 43 +++++++--- 5 files changed, 132 insertions(+), 49 deletions(-) diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index fe0c2a3991f0..e96f6531be6b 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -83,8 +83,8 @@ vllm serve Qwen/Qwen3.5-0.8B \ ``` This backend supports standard decoding and one to seven speculative tokens. -Omit `--speculative-config` for standard decoding. It requires BF16 state and -128-element key and value heads. Pipeline parallelism, microbatching, and KV +Omit `--speculative-config` for standard decoding. The default path uses BF16 state +and 128-element key and value heads. Pipeline parallelism, microbatching, and KV connectors are unsupported. Ragged scheduled rows are supported. Adaptive verification is unsupported because it changes row lengths on the GPU after the CPU metadata builder determines the output indices. @@ -101,6 +101,14 @@ missing kernels or change FlashInfer. Set `SGLANG_GDN_WY_STRIDED_QKV=1` before process startup. FlashInfer reads this setting when its kernel module is imported; setting it later is insufficient. +For speculative decoding on SM100-class GPUs, set +`--mamba-ssm-cache-dtype float16` to select FlashInfer 0.7.3's direct UMMA API. +Keep model activations and convolution state in BF16. This path supports at most +four value heads per key head. It requires `--mamba-cache-mode none` and disabled +prefix caching. FP16 standard decoding, prefix materialization, and Mamba cache +stochastic rounding are unsupported. +The state and rings retain the existing paged allocation. + ## Notes - MTP only works for model families that support MTP in vLLM. diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index b85898cf9ef0..1b0cd6c1e860 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -16,14 +16,21 @@ ) -@pytest.mark.parametrize("width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp")]) +@pytest.mark.parametrize( + "width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp"), (4, "umma")] +) def test_dependencies_import_only_selected_decode_api(monkeypatch, width, mode): """An MTP-only FlashInfer installation must not need the STP module.""" module = ModuleType("provider") kernel = lambda **kwargs: None - setattr(module, f"gated_delta_rule_{mode}_ucache_flush", kernel) - suffix = "stp" if mode == "stp" else "flush" - expected = f"flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_{suffix}" + if mode == "umma": + api_name = "gated_delta_rule_mtp_ucache_flush_umma" + expected = "flashinfer.gdn_kernels.gdn_replay_mtp_umma" + else: + api_name = f"gated_delta_rule_{mode}_ucache_flush" + suffix = "stp" if mode == "stp" else "flush" + expected = f"flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_{suffix}" + setattr(module, api_name, kernel) original_import = builtins.__import__ def import_provider(name, *args, **kwargs): @@ -36,7 +43,8 @@ def import_provider(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", import_provider) check_gdn_replayssm_dependencies.cache_clear() try: - assert check_gdn_replayssm_dependencies(width, False) is kernel + state_dtype = torch.float16 if mode == "umma" else torch.bfloat16 + assert check_gdn_replayssm_dependencies(width, False, state_dtype) is kernel finally: check_gdn_replayssm_dependencies.cache_clear() @@ -174,20 +182,33 @@ def _reconstruct_replayssm_state( reason="FlashInfer GDN ReplaySSM requires an SM90+ CUDA device", ) @pytest.mark.parametrize( - "width,query_len,accepted", - [(1, 1, 1), (4, 3, 2), (4, 4, 2), (8, 6, 4), (8, 8, 4)], + "width,query_len,accepted,state_dtype", + [ + (1, 1, 1, torch.bfloat16), + (4, 3, 2, torch.bfloat16), + (4, 4, 2, torch.bfloat16), + (8, 6, 4, torch.bfloat16), + (8, 8, 4, torch.bfloat16), + (4, 3, 2, torch.float16), + (8, 6, 4, torch.float16), + ], ) @torch.inference_mode() def test_gdn_replayssm_output_and_accepted_state( - monkeypatch, width, query_len, accepted + monkeypatch, width, query_len, accepted, state_dtype ): """Check nonzero outputs and accepted state through replay, wrap, and flush.""" monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") - module = ( - "gdn_decode_bf16_wy_ucache_stp" - if width == 1 - else "gdn_decode_bf16_wy_ucache_flush" - ) + if state_dtype == torch.float16: + if torch.cuda.get_device_capability()[0] != 10: + pytest.skip("FP16 GDN ReplaySSM requires an SM100-class CUDA device") + module = "gdn_replay_mtp_umma" + else: + module = ( + "gdn_decode_bf16_wy_ucache_stp" + if width == 1 + else "gdn_decode_bf16_wy_ucache_flush" + ) pytest.importorskip(f"flashinfer.gdn_kernels.{module}") device, dtype = torch.device("cuda"), torch.bfloat16 generator = torch.Generator(device=device).manual_seed(20260909 + width) @@ -203,7 +224,7 @@ def random_tensor(*shape): (1, ring_slots, 128), (2, ring_slots), ] - dtypes = [dtype, dtype, dtype, torch.float32] + dtypes = [state_dtype, dtype, dtype, torch.float32] sizes = [math.prod(shape) * dt.itemsize for shape, dt in zip(shapes, dtypes)] pages = torch.zeros(2, sum(sizes) + 16, dtype=torch.uint8, device=device) states = [] @@ -220,13 +241,15 @@ def random_tensor(*shape): dt_bias = torch.nn.Parameter(random_tensor(2) * 0.2) accepted_state = checkpoint[1].float().clone() padding_page = pages[0].clone() + capture = width == 1 or (state_dtype == torch.float16 and width == 4) graph = None # Eighteen steps cross the native STP flush and the MTP ring boundary. for step in range(18): - if width == 1 and step == 9: + if capture and step == 9: # Captured execution must observe an in-place weight reload. A_log.add_(2.0) + dt_bias.add_(1.0) q, k = random_tensor(query_len, 1, 128), random_tensor(query_len, 1, 128) v = random_tensor(query_len, 2, 128) * 0.1 a, b = random_tensor(query_len, 2) * 0.2, random_tensor(query_len, 2) @@ -237,12 +260,18 @@ def random_tensor(*shape): mixed_qkv = projection[:, : mixed_qkv.shape[1]] gates = torch.stack((a, b), dim=1) a, b = gates[:, 0], gates[:, 1] - if width == 1: + if capture: # Captured padding has input storage but no token or live state. - mixed_qkv = torch.cat((mixed_qkv, torch.ones_like(mixed_qkv))) + padding_tokens = 1 if width == 1 else width + mixed_qkv = torch.cat( + ( + mixed_qkv, + torch.ones_like(mixed_qkv[:1]).expand(padding_tokens, -1), + ) + ) a, b = ( - torch.cat((a, torch.ones_like(a))), - torch.cat((b, torch.ones_like(b))), + torch.cat((a, torch.ones_like(a[:1]).expand(padding_tokens, -1))), + torch.cat((b, torch.ones_like(b[:1]).expand(padding_tokens, -1))), ) inputs = dict( mixed_qkv=mixed_qkv, @@ -255,15 +284,17 @@ def random_tensor(*shape): ring_start=ring_start, num_committed=committed, query_start_loc=torch.tensor( - [0, 1, 1] if width == 1 else [0, query_len], + [0, query_len, query_len] if capture else [0, query_len], dtype=torch.int32, device=device, ), state_indices=torch.tensor( - [1, 0] if width == 1 else [1], dtype=torch.int32, device=device + [1, 0] if capture else [1], dtype=torch.int32, device=device ), - output_indices=torch.arange( - 2 if width == 1 else query_len, dtype=torch.int32, device=device + output_indices=torch.tensor( + list(range(query_len)) + ([width] if capture else []), + dtype=torch.int32, + device=device, ), executed_query_width=width, uniform_query_length=query_len, @@ -272,7 +303,7 @@ def random_tensor(*shape): head_k_dim=128, head_v_dim=128, ) - if width == 1: + if capture: if graph is None: graph_inputs = inputs saved_pages = pages.clone() @@ -290,10 +321,10 @@ def random_tensor(*shape): for name in ("mixed_qkv", "a", "b"): graph_inputs[name].copy_(inputs[name]) graph.replay() - output = graph_output[:1] - assert graph_output[1].eq(0).all() + output = graph_output[:query_len] + assert graph_output[query_len:].eq(0).all() torch.testing.assert_close(pages[0], padding_page, rtol=0, atol=0) - a, b = a[:1], b[:1] + a, b = a[:query_len], b[:query_len] else: output = run_gdn_replayssm(**inputs) expected_output, _ = _reference_gdn_step( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 6d868ce80635..8d6f901458c7 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -4016,7 +4016,7 @@ def validate_gdn_replayssm(self) -> "VllmConfig": "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" ) gdn_replayssm_geometry(self.num_speculative_tokens) - # This integration selects the provider's BF16 path. + # Both provider paths require BF16 activations and convolution state. if self.model_config is not None: if self.model_config.dtype != torch.bfloat16: raise ValueError( @@ -4035,10 +4035,27 @@ def validate_gdn_replayssm(self) -> "VllmConfig": if self.cache_config.mamba_ssm_cache_dtype not in ( "auto", "bfloat16", + "float16", ): raise ValueError( - "FlashInfer GDN ReplaySSM requires a bfloat16 recurrent state" + "FlashInfer GDN ReplaySSM requires BF16 or FP16 recurrent state" ) + if self.cache_config.mamba_ssm_cache_dtype == "float16": + if self.mamba_config.enable_stochastic_rounding: + raise ValueError( + "FP16 GDN ReplaySSM does not support " + "--enable-mamba-cache-stochastic-rounding" + ) + if self.num_speculative_tokens == 0: + raise ValueError("FP16 GDN ReplaySSM requires speculative decoding") + if ( + self.cache_config.mamba_cache_mode != "none" + or self.cache_config.enable_prefix_caching + ): + raise ValueError( + "FP16 GDN ReplaySSM requires --mamba-cache-mode=none " + "and disabled prefix caching" + ) # These execution modes need additional tracker lifecycle support. if self.parallel_config.pipeline_parallel_size > 1: raise ValueError( diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 357bc483445c..1da7a95be1c4 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -890,6 +890,7 @@ def __init__( check_gdn_replayssm_dependencies( self.replayssm_executed_query_width, needs_materializer=self.cache_config.mamba_cache_mode == "align", + state_dtype=self.get_state_dtype()[1], ) elif self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): reason = self._fused_gdn_decode_unsupported_reason(vllm_config) @@ -934,14 +935,17 @@ def _flashinfer_replayssm_unsupported_reason( return "value heads must be an integer multiple of key heads" if vllm_config.model_config.dtype != torch.bfloat16: return "the GDN ReplaySSM adapter requires bfloat16 model activations" - if ( - conv_state_dtype != torch.bfloat16 - or recurrent_state_dtype != torch.bfloat16 - ): - return ( - "the GDN ReplaySSM adapter requires bfloat16 convolution " - "and recurrent state" - ) + if conv_state_dtype != torch.bfloat16: + return "the GDN ReplaySSM adapter requires bfloat16 convolution state" + if recurrent_state_dtype == torch.float16: + if not current_platform.is_device_capability_family(100): + return "FP16 GDN ReplaySSM requires an SM100-class CUDA device" + if self.num_v_heads // self.num_k_heads > 4: + return ( + "FP16 GDN ReplaySSM supports at most four value heads per key head" + ) + elif recurrent_state_dtype != torch.bfloat16: + return "the GDN ReplaySSM adapter requires BF16 or FP16 recurrent state" if not current_platform.has_device_capability(90): return "the initial integration requires an SM90+ CUDA device" return None diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index 71e87bf4e93e..cbc55a7446f3 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -15,27 +15,43 @@ @cache def check_gdn_replayssm_dependencies( - executed_width: int, needs_materializer: bool + executed_width: int, + needs_materializer: bool, + state_dtype: torch.dtype = torch.bfloat16, ) -> Callable[..., torch.Tensor]: """Validate the environment and return only the selected decode API.""" if executed_width not in (1, 4, 8): raise ValueError(f"Unsupported GDN ReplaySSM executed width: {executed_width}") + if state_dtype not in (torch.bfloat16, torch.float16): + raise ValueError(f"Unsupported GDN ReplaySSM state dtype: {state_dtype}") + use_umma = state_dtype == torch.float16 + if use_umma and (executed_width == 1 or needs_materializer): + raise ValueError( + "FP16 GDN ReplaySSM requires speculative decoding without prefix caching" + ) if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": raise RuntimeError( "FlashInfer GDN ReplaySSM requires " "SGLANG_GDN_WY_STRIDED_QKV=1 before importing FlashInfer" ) - api = ( - "gated_delta_rule_stp_ucache_flush" - if executed_width == 1 - else "gated_delta_rule_mtp_ucache_flush" - ) + if use_umma: + api = "gated_delta_rule_mtp_ucache_flush_umma" + else: + api = ( + "gated_delta_rule_stp_ucache_flush" + if executed_width == 1 + else "gated_delta_rule_mtp_ucache_flush" + ) mode = f"executed width={executed_width}, cache mode=" + ( "align" if needs_materializer else "none" ) try: # MTP-only providers need not ship the separate STP API. - if executed_width == 1: + if use_umma: + from flashinfer.gdn_kernels.gdn_replay_mtp_umma import ( + gated_delta_rule_mtp_ucache_flush_umma as kernel, + ) + elif executed_width == 1: from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_stp import ( gated_delta_rule_stp_ucache_flush as kernel, ) @@ -156,10 +172,15 @@ def run_gdn_replayssm( raise ValueError( "FlashInfer GDN ReplaySSM requires exactly three auxiliary rings" ) - if mixed_qkv.dtype != torch.bfloat16 or checkpoint_state.dtype != torch.bfloat16: + if mixed_qkv.dtype != torch.bfloat16 or checkpoint_state.dtype not in ( + torch.bfloat16, + torch.float16, + ): raise ValueError( - "FlashInfer GDN ReplaySSM requires bfloat16 inputs and checkpoint state" + "FlashInfer GDN ReplaySSM requires bfloat16 inputs and BF16 or FP16 state" ) + if checkpoint_state.dtype == torch.float16 and executed_query_width == 1: + raise ValueError("FP16 GDN ReplaySSM requires speculative decoding") # Native STP's public contract is T=1, but its serving fast path consumes a # caller-owned zero-padded T=4 buffer. Packing a shared QKV allocation here @@ -235,7 +256,9 @@ def run_gdn_replayssm( ) if kernel is None: - kernel = check_gdn_replayssm_dependencies(executed_query_width, False) + kernel = check_gdn_replayssm_dependencies( + executed_query_width, False, checkpoint_state.dtype + ) if executed_query_width == 1: # STP updates these cursor copies. Every layer must start from the # same pre-step values; model-wide postprocess publishes the update. From b95a484d72ba1bc1b75a18a8c1e17560f243f812 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Fri, 6 Nov 2026 11:32:20 +0100 Subject: [PATCH 06/14] [Mamba] Enable FP16 GDN ReplaySSM STP and prefix caching Use the released FlashInfer UMMA API for standard decoding with a 32-slot ring and caller-owned cursor updates. Select the prefix materializer by the actual state dtype and reject incompatible import-time selection. Preserve the BF16 provider contracts and cache-mode restrictions. Extend the existing recurrence and postprocess cases for FP16 standard decoding. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/speculative_decoding/mtp.md | 19 +++++--- tests/kernels/mamba/test_gdn_replayssm.py | 44 +++++++++++++++++-- tests/kernels/mamba/test_ssu_dispatch.py | 28 +++++++----- vllm/config/vllm.py | 10 ++--- vllm/model_executor/layers/mamba/gdn/base.py | 10 +++-- .../layers/mamba/gdn/replayssm.py | 28 +++++------- .../layers/mamba/mamba_utils.py | 8 ++-- .../layers/mamba/ops/ssu_dispatch.py | 13 +++++- vllm/model_executor/models/qwen3_5.py | 4 +- .../model_executor/warmup/replayssm_warmup.py | 2 +- vllm/v1/worker/replayssm_utils.py | 12 ++++- 11 files changed, 124 insertions(+), 54 deletions(-) diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index e96f6531be6b..cee6848256a3 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -91,23 +91,28 @@ CPU metadata builder determines the output indices. Set `enable_adaptive_verification` to `false` in the speculative configuration. Prefix caching requires `--mamba-cache-mode align`. -Standard decoding requires `gated_delta_rule_stp_ucache_flush`. Speculative +BF16 standard decoding requires `gated_delta_rule_stp_ucache_flush`. Speculative decoding requires `gated_delta_rule_mtp_ucache_flush`, available in FlashInfer 0.7.2.post1. Prefix caching also requires `gdn_prefix_materialize`, available in -FlashInfer 0.7.3. The STP API is not included in either release, so standard +FlashInfer 0.7.3. The STP API is not included in either release, so BF16 standard decoding requires a compatible provider build. This integration does not install missing kernels or change FlashInfer. Set `SGLANG_GDN_WY_STRIDED_QKV=1` before process startup. FlashInfer reads this setting when its kernel module is imported; setting it later is insufficient. -For speculative decoding on SM100-class GPUs, set +For standard or speculative decoding on SM100-class GPUs, set `--mamba-ssm-cache-dtype float16` to select FlashInfer 0.7.3's direct UMMA API. Keep model activations and convolution state in BF16. This path supports at most -four value heads per key head. It requires `--mamba-cache-mode none` and disabled -prefix caching. FP16 standard decoding, prefix materialization, and Mamba cache -stochastic rounding are unsupported. -The state and rings retain the existing paged allocation. +four value heads per key head. It uses a 32-slot ring for both decoding modes. +The state and rings retain the existing paged allocation. Mamba cache stochastic +rounding is unsupported. + +For FP16 prefix caching, use `--mamba-cache-mode align` and enable prefix caching. +Set `GDN_UCACHE_STATE_DTYPE=fp16` before process startup in each worker environment. +FlashInfer selects the materializer's state dtype during import. Changing the +environment after import does not change that dtype. BF16 prefix caching requires +the default BF16 materializer, so use separate processes for different state dtypes. ## Notes diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 1b0cd6c1e860..139bea97f584 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -14,10 +14,13 @@ pack_replayssm_rows, run_gdn_replayssm, ) +from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( + _load_gdn_replayssm_materialize, +) @pytest.mark.parametrize( - "width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp"), (4, "umma")] + "width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp"), (1, "umma"), (4, "umma")] ) def test_dependencies_import_only_selected_decode_api(monkeypatch, width, mode): """An MTP-only FlashInfer installation must not need the STP module.""" @@ -49,6 +52,37 @@ def import_provider(name, *args, **kwargs): check_gdn_replayssm_dependencies.cache_clear() +@pytest.mark.parametrize("loaded_dtype", [torch.bfloat16, torch.float16]) +def test_materializer_rejects_dtype_change_after_import(monkeypatch, loaded_dtype): + """Changing the environment cannot change an imported provider's state dtype.""" + provider = ModuleType("provider") + materialize = lambda **kwargs: None + monkeypatch.setattr(provider, "gdn_prefix_materialize", materialize, raising=False) + monkeypatch.setattr(provider, "state_ty_torch", lambda: loaded_dtype, raising=False) + original_import = builtins.__import__ + + def import_provider(name, *args, **kwargs): + if name == "flashinfer.gdn_kernels.gdn_prefix_materialize": + return provider + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_provider) + _load_gdn_replayssm_materialize.cache_clear() + try: + assert _load_gdn_replayssm_materialize(loaded_dtype) is materialize + requested_dtype = ( + torch.float16 if loaded_dtype == torch.bfloat16 else torch.bfloat16 + ) + monkeypatch.setenv( + "GDN_UCACHE_STATE_DTYPE", + "fp16" if requested_dtype == torch.float16 else "bf16", + ) + with pytest.raises(ValueError, match="before process startup"): + _load_gdn_replayssm_materialize(requested_dtype) + finally: + _load_gdn_replayssm_materialize.cache_clear() + + def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): values = torch.tensor([[1.0], [2.0], [3.0]]) packed = pack_replayssm_rows( @@ -189,6 +223,7 @@ def _reconstruct_replayssm_state( (4, 4, 2, torch.bfloat16), (8, 6, 4, torch.bfloat16), (8, 8, 4, torch.bfloat16), + (1, 1, 1, torch.float16), (4, 3, 2, torch.float16), (8, 6, 4, torch.float16), ], @@ -212,7 +247,8 @@ def test_gdn_replayssm_output_and_accepted_state( pytest.importorskip(f"flashinfer.gdn_kernels.{module}") device, dtype = torch.device("cuda"), torch.bfloat16 generator = torch.Generator(device=device).manual_seed(20260909 + width) - ring_slots = 16 if width == 1 else 32 + native_stp = width == 1 and state_dtype == torch.bfloat16 + ring_slots = 16 if native_stp else 32 def random_tensor(*shape): return torch.randn(*shape, generator=generator, device=device).to(dtype) @@ -236,6 +272,8 @@ def random_tensor(*shape): pages[0].fill_(17) # Nonzero canaries catch writes to the padding page. checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) ring_start = torch.zeros(2, dtype=torch.int32, device=device) + if width == 1 and not native_stp: + ring_start[1] = 24 # Cross the UMMA ring boundary before the first flush. committed = torch.zeros_like(ring_start) A_log = torch.nn.Parameter(torch.full((2,), -3.0, device=device)) dt_bias = torch.nn.Parameter(random_tensor(2) * 0.2) @@ -342,7 +380,7 @@ def random_tensor(*shape): dt_bias, ) old_committed = int(committed[1].item()) - if width == 1: + if native_stp: committed[1] = 0 if old_committed >= 15 else old_committed + 1 elif old_committed + width > 16: ring_start[1] = (ring_start[1] + old_committed) % ring_slots diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index e388257baee0..2472d5e752c8 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -316,21 +316,24 @@ def test_replayssm_materializes_only_accepted_boundary_with_compacted_rows( @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("post_step", [False, True], ids=["v1", "v2"]) -@pytest.mark.parametrize("gdn_stp", [False, True]) +@pytest.mark.parametrize("gdn_mode", ["none", "stp", "umma_stp"]) def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( - post_step, gdn_stp + post_step, gdn_mode ): """Skip prefill self-copies, reset trackers, and preserve in-place MTP replay.""" + gdn_stp = gdn_mode == "stp" + umma_stp = gdn_mode == "umma_stp" + ring_slots = 16 if gdn_stp else 32 if umma_stp else 20 mixer = _replayssm_mixer( kv_cache=( torch.empty(0, device="cuda"), torch.empty(6, 1, 1, 1, device="cuda"), - torch.empty(6, 1, 20, 1, device="cuda"), - torch.empty(6, 1, 20, device="cuda"), - torch.empty(6, 1, 20, 1, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), + torch.empty(6, 1, ring_slots, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), ), ring_start=_cuda_ints([3] * 6), - committed=_cuda_ints([2, 2, 2, 2, 15 if gdn_stp else 2, 2]), + committed=_cuda_ints([2, 2, 2, 2, 15 if gdn_stp else 16 if umma_stp else 2, 2]), A=torch.empty(1, device="cuda"), ) spec = MambaSpec( @@ -350,7 +353,7 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( context = ReplaySSMModelContext.create([([mixer], spec, table)], max_num_reqs=4) # Reordered rows: canonical prefill in slot 2; in-place MTP replay in slot 4; # padded slot; padded request. - query_len = 1 if gdn_stp else 4 + query_len = 1 if gdn_stp or umma_stp else 4 _postprocess_replayssm_kernel[(4, 1)]( _cuda_ints([1, 2, 0, -1]), _cuda_ints([0, 48, 48 + query_len, 64 + query_len, 64 + query_len]) @@ -372,14 +375,14 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( MAX_NUM_REQS=4, MAMBA_BLOCK_SIZE=16, LOGICAL_WINDOW=16, - RING_BUFFER_LEN=16 if gdn_stp else 20, + RING_BUFFER_LEN=ring_slots, PAD_SLOT_ID=NULL_BLOCK_ID, QUERY_METADATA_IS_CUMULATIVE=post_step, NUM_COMPUTED_IS_POST_STEP=post_step, HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, - EXECUTED_QUERY_WIDTH=1 if gdn_stp else 0, + EXECUTED_QUERY_WIDTH=1 if gdn_stp or umma_stp else 0, GDN_STP=gdn_stp, ) _compact_replayssm_requests_kernel[(1,)]( @@ -391,7 +394,12 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( BLOCK_SIZE=4, ) - assert context.plan_flush_count.tolist() == [-1, 0 if gdn_stp else 6, -1, -1] + assert context.plan_flush_count.tolist() == [ + -1, + 0 if gdn_stp else 1 if umma_stp else 6, + -1, + -1, + ] assert context.active_request_indices.tolist() == [1, -1, -1, -1] assert context.src_slots[0, 1].item() == 4 assert context.dst_slots[0, 1].item() == 4 diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 8d6f901458c7..2a19dbdced24 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -4046,15 +4046,13 @@ def validate_gdn_replayssm(self) -> "VllmConfig": "FP16 GDN ReplaySSM does not support " "--enable-mamba-cache-stochastic-rounding" ) - if self.num_speculative_tokens == 0: - raise ValueError("FP16 GDN ReplaySSM requires speculative decoding") if ( - self.cache_config.mamba_cache_mode != "none" - or self.cache_config.enable_prefix_caching + self.cache_config.mamba_cache_mode == "align" + and os.environ.get("GDN_UCACHE_STATE_DTYPE") != "fp16" ): raise ValueError( - "FP16 GDN ReplaySSM requires --mamba-cache-mode=none " - "and disabled prefix caching" + "FP16 GDN ReplaySSM prefix materialization requires " + "GDN_UCACHE_STATE_DTYPE=fp16 before process startup" ) # These execution modes need additional tracker lifecycle support. if self.parallel_config.pipeline_parallel_size > 1: diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 3329dcc0b854..ddd7fdc32775 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -57,12 +57,16 @@ def __init__( self.replayssm_ring_slots: int | None = None if self.use_replayssm: self.replayssm_executed_query_width, self.replayssm_ring_slots = ( - gdn_replayssm_geometry(self.num_spec) + gdn_replayssm_geometry(self.num_spec, self.get_state_dtype()[1]) ) if self.replayssm_executed_query_width is not None: - # STP accepts four input positions: one real token and three zeros. + # BF16 STP accepts one real token and three zero input positions. # These offsets pack token inputs; they are not replay-ring cursors. - packed_width = max(self.replayssm_executed_query_width, 4) + packed_width = ( + 4 + if self.replayssm_ring_slots == 16 + else self.replayssm_executed_query_width + ) self.register_buffer( "_replayssm_offsets", torch.arange(packed_width, dtype=torch.int32), diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index cbc55a7446f3..f20fdb1b96cf 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -25,10 +25,6 @@ def check_gdn_replayssm_dependencies( if state_dtype not in (torch.bfloat16, torch.float16): raise ValueError(f"Unsupported GDN ReplaySSM state dtype: {state_dtype}") use_umma = state_dtype == torch.float16 - if use_umma and (executed_width == 1 or needs_materializer): - raise ValueError( - "FP16 GDN ReplaySSM requires speculative decoding without prefix caching" - ) if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": raise RuntimeError( "FlashInfer GDN ReplaySSM requires " @@ -69,7 +65,7 @@ def check_gdn_replayssm_dependencies( ) try: - _load_gdn_replayssm_materialize() + _load_gdn_replayssm_materialize(state_dtype) except ImportError as e: raise ImportError( f"FlashInfer GDN ReplaySSM ({mode}) requires " @@ -159,8 +155,8 @@ def run_gdn_replayssm( ) -> torch.Tensor: """Pack rows, invoke FlashInfer, and return only real-token outputs. - STP receives one real row per request. The adapter supplies FlashInfer's - four-token input buffer: one real token followed by three zero rows. + BF16 STP receives one real token followed by three zero input positions. + FP16 UMMA receives the executed width, including one for standard decoding. vLLM publishes cursors once per model step, after every layer runs. """ if executed_query_width not in (1, 4, 8): @@ -179,18 +175,16 @@ def run_gdn_replayssm( raise ValueError( "FlashInfer GDN ReplaySSM requires bfloat16 inputs and BF16 or FP16 state" ) - if checkpoint_state.dtype == torch.float16 and executed_query_width == 1: - raise ValueError("FP16 GDN ReplaySSM requires speculative decoding") - - # Native STP's public contract is T=1, but its serving fast path consumes a + native_stp = executed_query_width == 1 and checkpoint_state.dtype == torch.bfloat16 + # BF16 STP's public contract is T=1, but its serving fast path consumes a # caller-owned zero-padded T=4 buffer. Packing a shared QKV allocation here # preserves the common token stride required by block-strided cache pools. - packed_width = 4 if executed_query_width == 1 else executed_query_width - uniform = executed_query_width > 1 and uniform_query_length == packed_width + packed_width = 4 if native_stp else executed_query_width + uniform = not native_stp and uniform_query_length == packed_width # STP metadata permits only zero or one token per row. Eager zero-length # rows use the packer's fallback. Captured padding can have input storage; # its negative state index makes FlashInfer skip that row. - single_token = executed_query_width == 1 + single_token = native_stp packed_qkv = pack_replayssm_rows( mixed_qkv, query_start_loc, @@ -208,7 +202,7 @@ def run_gdn_replayssm( # eight. Give each gate its own aligned allocation instead. # The MTP kernel advances every padded lane. Make those lanes exact # recurrent no-ops: a=-inf gives zero decay after softplus(a + dt_bias), - # while zero q/k/v and b give a zero state update. STP has no ragged real + # while zero q/k/v and b give a zero state update. BF16 STP has no ragged real # rows, so only invalid graph-padding requests can be padded here; their # negative state index skips the kernel CTA. STP requires three zero # input rows after the real token. It returns and commits only that token. @@ -217,7 +211,7 @@ def run_gdn_replayssm( query_start_loc, packed_width, offsets, - padding_value=0.0 if executed_query_width == 1 else -torch.inf, + padding_value=0.0 if native_stp else -torch.inf, uniform=uniform, single_token=single_token, ) @@ -259,7 +253,7 @@ def run_gdn_replayssm( kernel = check_gdn_replayssm_dependencies( executed_query_width, False, checkpoint_state.dtype ) - if executed_query_width == 1: + if native_stp: # STP updates these cursor copies. Every layer must start from the # same pre-step values; model-wide postprocess publishes the update. mode_kwargs = {"prepadded": True} diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index d39dc156a46b..10e59e234577 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -28,15 +28,17 @@ ConvStateLayoutType = Literal["SD", "DS"] -def gdn_replayssm_geometry(num_speculative_tokens: int) -> tuple[int, int]: +def gdn_replayssm_geometry( + num_speculative_tokens: int, state_dtype: torch.dtype = torch.bfloat16 +) -> tuple[int, int]: """Return the selected GDN provider's query width and physical ring slots. Both paths retain at most 16 logical history records. MTP needs extra physical slots for verification writes while other blocks read history. - STP appends one record or folds the current token without appending. + BF16 STP folds the current token. FP16 UMMA STP uses the MTP ring contract. """ if num_speculative_tokens == 0: - return 1, 16 + return 1, 32 if state_dtype == torch.float16 else 16 if 1 <= num_speculative_tokens <= 3: return 4, 32 if 4 <= num_speculative_tokens <= 7: diff --git a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py index 55ebe8fe565e..bebadd4e8ae3 100644 --- a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py +++ b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py @@ -452,16 +452,27 @@ def selective_state_update_replayssm_flashinfer( @cache -def _load_gdn_replayssm_materialize() -> Callable[..., None]: +def _load_gdn_replayssm_materialize( + state_dtype: torch.dtype = torch.bfloat16, +) -> Callable[..., None]: try: from flashinfer.gdn_kernels.gdn_prefix_materialize import ( gdn_prefix_materialize, + state_ty_torch, ) except ImportError as e: raise ImportError( "FlashInfer GDN ReplaySSM prefix caching requires " "flashinfer.gdn_kernels.gdn_prefix_materialize" ) from e + if state_ty_torch() != state_dtype: + dtype_name = "fp16" if state_dtype == torch.float16 else "bf16" + raise ValueError( + f"FlashInfer GDN prefix materialization loaded state dtype " + f"{state_ty_torch()}, but vLLM requires {state_dtype}. Set " + f"GDN_UCACHE_STATE_DTYPE={dtype_name} before process startup. " + "Changing the environment after importing FlashInfer is insufficient." + ) return gdn_prefix_materialize diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 695589e97480..a459fec5e91f 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -428,7 +428,9 @@ def get_mamba_state_shape_from_config( ) if vllm_config.is_gdn_replayssm_enabled(): - _, ring_slots = gdn_replayssm_geometry(num_spec) + _, ring_slots = gdn_replayssm_geometry( + num_spec, cls.get_mamba_state_dtype_from_config(vllm_config)[1] + ) shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( tp_size, hf_config.linear_num_key_heads, diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index 3bb1f31e58e6..f66ac9babb91 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -256,7 +256,6 @@ def _warm_gdn_replayssm_materializer( if runner.vllm_config.cache_config.mamba_cache_mode != "align": return - materialize = _load_gdn_replayssm_materialize() warmed: set[tuple[Any, ...]] = set() for module in runner.get_model().modules(): if not getattr(module, "use_flashinfer_replayssm", False) or getattr( @@ -279,6 +278,7 @@ def _warm_gdn_replayssm_materializer( ) if signature in warmed: continue + materialize = _load_gdn_replayssm_materialize(state.dtype) warmed.add(signature) inactive = torch.full( (max_num_reqs,), -1, dtype=torch.int32, device=state.device diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index c8fbaf175263..538fb748858c 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -457,7 +457,12 @@ def create( "executed query width 1, 4, or 8; got " f"window={logical_window}, width={executed_query_width}" ) - expected_ring_buffer_len = 16 if executed_query_width == 1 else 32 + expected_ring_buffer_len = ( + 16 + if executed_query_width == 1 + and first.kv_cache[1].dtype == torch.bfloat16 + else 32 + ) if ring_buffer_len != expected_ring_buffer_len: raise ValueError( "FlashInfer GDN ReplaySSM ring depth does not match its " @@ -560,7 +565,9 @@ def create( def _materialize(self) -> None: """Publish canonical prefix snapshots with the selected backend.""" if self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: - self.materialize_gdn(_load_gdn_replayssm_materialize()) + self.materialize_gdn( + _load_gdn_replayssm_materialize(self.mixers[0].kv_cache[1].dtype) + ) return assert self.materialize_tables is not None first = self.mixers[0] @@ -680,6 +687,7 @@ def postprocess( GDN_STP=( self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN and self.executed_query_width == 1 + and self.ring_buffer_len == 16 ), ) if not self.materialize_prefixes or not may_materialize: From c350280dacca0cd89fafdd72e2c087b984fc4a4c Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Fri, 6 Nov 2026 15:24:36 +0100 Subject: [PATCH 07/14] [Mamba] Warm BF16 ReplaySSM eager layouts and tactics Warm both supported QKV layouts at each reachable HMMA tactic. Use null state slots and preserve the existing temporary-state cleanup. Require the provider capability for dynamic strided shapes so released FlashInfer 0.7.3 keeps its existing warmup behavior. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/speculative_decoding/mtp.md | 7 ++ .../model_executor/warmup/replayssm_warmup.py | 92 ++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index cee6848256a3..c7c8c302b8d9 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -98,6 +98,13 @@ FlashInfer 0.7.3. The STP API is not included in either release, so BF16 standar decoding requires a compatible provider build. This integration does not install missing kernels or change FlashInfer. +FlashInfer 0.7.3 specializes the BF16 speculative decoder for batch and pool sizes. +Avoiding those recompilations requires a patched provider that declares +`SUPPORTS_DYNAMIC_STRIDED_SHAPES = True`. With that provider, startup also warms the +reachable input layouts and launch tactics. The extra warmup stays disabled for +providers without this capability. The prefix materializer has separate +pool-size specialization, which this decoder fix does not remove. + Set `SGLANG_GDN_WY_STRIDED_QKV=1` before process startup. FlashInfer reads this setting when its kernel module is imported; setting it later is insufficient. diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index f66ac9babb91..511893dbb791 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os from collections.abc import Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -298,8 +299,96 @@ def _warm_gdn_replayssm_materializer( ) +def _warm_gdn_replayssm_mixed_decode( + runner: "GPUModelRunner", executed_query_width: int +) -> None: + """Warm BF16 MTP layouts and launch tactics reachable outside full graphs.""" + if executed_query_width == 1: + return + from flashinfer.gdn_kernels.device_target import gdn_device_target + + from vllm.model_executor.layers.mamba.gdn.replayssm import run_gdn_replayssm + + max_num_reqs = min(runner.scheduler_config.max_num_seqs, runner.max_num_tokens) + warmed: set[tuple[Any, ...]] = set() + for module in runner.get_model().modules(): + if ( + not getattr(module, "use_flashinfer_replayssm", False) + or getattr(module, "replayssm_executed_query_width", None) + != executed_query_width + ): + continue + state, u_cache, k_cache, g_cache = module.kv_cache[1:5] + if state.dtype != torch.bfloat16: + continue + from flashinfer.gdn_kernels import gdn_decode_bf16_wy_ucache_flush as provider + + if not getattr(provider, "SUPPORTS_DYNAMIC_STRIDED_SHAPES", False): + return + signature = tuple( + (t.device, t.dtype, t.shape[1:], t.stride()) + for t in (state, u_cache, k_cache, g_cache) + ) + if signature in warmed: + continue + warmed.add(signature) + _, num_v_heads, head_v_dim, head_k_dim = state.shape + num_k_heads = k_cache.shape[1] + qkv_width = 2 * num_k_heads * head_k_dim + num_v_heads * head_v_dim + strides = [qkv_width] # Mixed requests and ragged rows produce compact QKV. + if runner.vllm_config.num_speculative_tokens + 1 == executed_query_width: + strides.append(qkv_width + num_v_heads * head_v_dim) # QKVZ view. + batches = {1} + if int(os.environ.get("GDN_WY_MBP") or 0) <= 0: + # FlashInfer 0.7.3 HMMA policy: min(ceil(HV * B / SM) + 1, 7). + # No provider selector exposes it. Warm one batch per reachable tactic. + num_sms = gdn_device_target(state.device).num_sms + batches.update(i * num_sms // num_v_heads + 1 for i in range(1, 6)) + for num_reqs in sorted(b for b in batches if b <= max_num_reqs): + num_tokens = num_reqs * executed_query_width + inactive = torch.zeros(num_reqs, dtype=torch.int32, device=state.device) + query_start = torch.arange( + 0, + num_tokens + 1, + executed_query_width, + dtype=torch.int32, + device=state.device, + ) + gates = torch.zeros( + num_tokens, num_v_heads, dtype=torch.bfloat16, device=state.device + ) + for stride in strides: + qkv = torch.zeros( + num_tokens, stride, dtype=torch.bfloat16, device=state.device + )[:, :qkv_width] + # Null slots become -1 in the adapter. The provider exits before + # reading or writing state, including uncommitted cache pages. + run_gdn_replayssm( + mixed_qkv=qkv, + a=gates, + b=gates, + A_log=module.A_log, + dt_bias=module.dt_bias, + checkpoint_state=state, + replayssm_cache=(u_cache, k_cache, g_cache), + ring_start=module._replayssm_ring_start, + num_committed=module._replayssm_prev_num_accepted, + query_start_loc=query_start, + state_indices=inactive, + output_indices=torch.empty( + 0, dtype=torch.long, device=state.device + ), + executed_query_width=executed_query_width, + num_k_heads=num_k_heads, + num_v_heads=num_v_heads, + head_k_dim=head_k_dim, + head_v_dim=head_v_dim, + uniform_query_length=executed_query_width, + ) + + def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: - """Compile GDN replay for every intended uniform-decode graph bucket.""" + """Compile GDN replay graph buckets and eager layout/tactic combinations.""" config = runner.vllm_config if not config.is_gdn_replayssm_enabled(): return @@ -358,4 +447,5 @@ def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: with _temporary_replayssm_autotune_state(runner, max_num_reqs): for num_reqs in sorted(capture_reqs): runner._dummy_run(num_tokens=num_reqs * query_len, **decode_kwargs) + _warm_gdn_replayssm_mixed_decode(runner, executed_query_width) _warm_gdn_replayssm_materializer(runner, max_num_reqs) From 218046b5c0fd2f2d0fd938e1706c3b8c736eb58b Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Fri, 6 Nov 2026 17:06:03 +0100 Subject: [PATCH 08/14] [Mamba] Restrict GDN ReplaySSM to FP16 UMMA behind --use-replayssm Remove the BF16 decode paths. BF16 standard decoding needed gated_delta_rule_stp_ucache_flush, which no FlashInfer release ships, and BF16 MTP compiled a new kernel for each batch and pool size. FP16 state runs FlashInfer 0.7.3's UMMA kernel for standard and speculative decoding. --use-replayssm alone now selects GDN ReplaySSM on Qwen3.5. VLLM_GDN_DECODE_KERNEL drops its flashinfer_replayssm choice, and vLLM sets GDN_UCACHE_STATE_DTYPE=fp16 for the prefix materializer before FlashInfer is imported. Share the replay prefill rule between the GDN metadata builder and V1 graph dispatch, test the GDN fold rule in postprocess, and move the docs next to the Mamba2 ReplaySSM section. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Andrii Skliar --- docs/features/automatic_prefix_caching.md | 22 ++ docs/features/speculative_decoding/mtp.md | 54 ---- tests/kernels/mamba/test_gdn_replayssm.py | 154 +++-------- tests/kernels/mamba/test_ssu_dispatch.py | 58 ++-- tests/test_config.py | 45 +++ tests/test_envs.py | 10 +- .../v1/attention/test_gdn_metadata_builder.py | 3 +- tests/v1/worker/test_gpu_model_runner.py | 6 +- vllm/config/cache.py | 7 +- vllm/config/vllm.py | 211 ++++++-------- vllm/envs.py | 26 +- vllm/model_executor/layers/mamba/gdn/base.py | 26 +- .../layers/mamba/gdn/qwen_gdn_linear_attn.py | 70 ++--- .../layers/mamba/gdn/replayssm.py | 260 ++++++------------ .../layers/mamba/mamba_utils.py | 54 ++-- .../layers/mamba/ops/ssu_dispatch.py | 25 -- vllm/model_executor/models/qwen3_5.py | 5 - .../model_executor/warmup/replayssm_warmup.py | 141 ++-------- vllm/v1/attention/backend.py | 4 +- vllm/v1/attention/backends/gdn_attn.py | 78 ++++-- .../worker/gpu/model_states/mamba_hybrid.py | 5 +- vllm/v1/worker/gpu_model_runner.py | 34 ++- vllm/v1/worker/mamba_utils.py | 8 +- vllm/v1/worker/replayssm_utils.py | 122 +++----- 24 files changed, 538 insertions(+), 890 deletions(-) diff --git a/docs/features/automatic_prefix_caching.md b/docs/features/automatic_prefix_caching.md index adab08e0d22b..18febf564054 100644 --- a/docs/features/automatic_prefix_caching.md +++ b/docs/features/automatic_prefix_caching.md @@ -65,6 +65,28 @@ FlashInfer ReplaySSM has these limits: Prefix reuse still requires a shared prefix with a saved Mamba checkpoint. The matching rules for hybrid Mamba models above also apply. +#### Qwen3.5 Gated DeltaNet + +On Qwen3.5 models, `--use-replayssm` applies ReplaySSM to the Gated DeltaNet layers. It runs FlashInfer's UMMA kernel, which requires FlashInfer 0.7.3 or later, an SM100-class GPU, and FP16 recurrent state: + +```bash +vllm serve nvidia/Qwen3.5-122B-A10B-NVFP4 \ + --dtype bfloat16 \ + --mamba-ssm-cache-dtype float16 \ + --use-replayssm \ + --enable-prefix-caching \ + --speculative-config '{"method":"mtp","num_speculative_tokens":7}' +``` + +Remove `--speculative-config` for standard decoding. Qwen3.5 Gated DeltaNet ReplaySSM has these limits: + +- Model activations and the convolution state must be BF16, with 128-element key and value heads and at most four value heads per key head. +- Speculative decoding supports up to seven draft tokens. Adaptive verification is not supported; set `enable_adaptive_verification` to `false`. +- `--replayssm-buffer-len` must be 16, the default. +- Mamba cache stochastic rounding, pipeline parallelism, microbatching, and KV connectors are not supported. + +With prefix caching, vLLM sets `GDN_UCACHE_STATE_DTYPE=fp16` before starting its workers, because FlashInfer reads it when first imported. If your own code imports FlashInfer before creating the engine, set this variable before that import. + ## Example workloads We describe two example workloads, where APC can provide huge performance benefit: diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index c7c8c302b8d9..34d8f9539b74 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -67,60 +67,6 @@ vllm serve XiaomiMiMo/MiMo-7B-Base \ --speculative-config '{"method":"mtp","num_speculative_tokens":1}' ``` -## Qwen3.5 GDN ReplaySSM - -GDN ReplaySSM stores recent recurrent inputs and reconstructs state from checkpoints. -This optional FlashInfer backend is disabled by default. Enable it for Qwen3.5 with: - -```bash -VLLM_USE_V2_MODEL_RUNNER=1 SGLANG_GDN_WY_STRIDED_QKV=1 \ -VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm \ -vllm serve Qwen/Qwen3.5-0.8B \ - --dtype bfloat16 --mamba-cache-dtype bfloat16 \ - --mamba-ssm-cache-dtype bfloat16 --use-replayssm \ - --replayssm-buffer-len 16 --no-enable-prefix-caching \ - --speculative-config '{"method":"mtp","num_speculative_tokens":3}' -``` - -This backend supports standard decoding and one to seven speculative tokens. -Omit `--speculative-config` for standard decoding. The default path uses BF16 state -and 128-element key and value heads. Pipeline parallelism, microbatching, and KV -connectors are unsupported. Ragged scheduled rows are supported. Adaptive -verification is unsupported because it changes row lengths on the GPU after the -CPU metadata builder determines the output indices. -Set `enable_adaptive_verification` to `false` in the speculative configuration. -Prefix caching requires `--mamba-cache-mode align`. - -BF16 standard decoding requires `gated_delta_rule_stp_ucache_flush`. Speculative -decoding requires `gated_delta_rule_mtp_ucache_flush`, available in FlashInfer -0.7.2.post1. Prefix caching also requires `gdn_prefix_materialize`, available in -FlashInfer 0.7.3. The STP API is not included in either release, so BF16 standard -decoding requires a compatible provider build. This integration does not install -missing kernels or change FlashInfer. - -FlashInfer 0.7.3 specializes the BF16 speculative decoder for batch and pool sizes. -Avoiding those recompilations requires a patched provider that declares -`SUPPORTS_DYNAMIC_STRIDED_SHAPES = True`. With that provider, startup also warms the -reachable input layouts and launch tactics. The extra warmup stays disabled for -providers without this capability. The prefix materializer has separate -pool-size specialization, which this decoder fix does not remove. - -Set `SGLANG_GDN_WY_STRIDED_QKV=1` before process startup. FlashInfer reads this -setting when its kernel module is imported; setting it later is insufficient. - -For standard or speculative decoding on SM100-class GPUs, set -`--mamba-ssm-cache-dtype float16` to select FlashInfer 0.7.3's direct UMMA API. -Keep model activations and convolution state in BF16. This path supports at most -four value heads per key head. It uses a 32-slot ring for both decoding modes. -The state and rings retain the existing paged allocation. Mamba cache stochastic -rounding is unsupported. - -For FP16 prefix caching, use `--mamba-cache-mode align` and enable prefix caching. -Set `GDN_UCACHE_STATE_DTYPE=fp16` before process startup in each worker environment. -FlashInfer selects the materializer's state dtype during import. Changing the -environment after import does not change that dtype. BF16 prefix caching requires -the default BF16 materializer, so use separate processes for different state dtypes. - ## Notes - MTP only works for model families that support MTP in vLLM. diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 139bea97f584..387c60557dcb 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -10,51 +10,16 @@ import torch.nn.functional as F from vllm.model_executor.layers.mamba.gdn.replayssm import ( - check_gdn_replayssm_dependencies, + load_gdn_replayssm_kernel, + load_gdn_replayssm_materializer, pack_replayssm_rows, run_gdn_replayssm, ) -from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( - _load_gdn_replayssm_materialize, -) - - -@pytest.mark.parametrize( - "width,mode", [(1, "stp"), (4, "mtp"), (8, "mtp"), (1, "umma"), (4, "umma")] -) -def test_dependencies_import_only_selected_decode_api(monkeypatch, width, mode): - """An MTP-only FlashInfer installation must not need the STP module.""" - module = ModuleType("provider") - kernel = lambda **kwargs: None - if mode == "umma": - api_name = "gated_delta_rule_mtp_ucache_flush_umma" - expected = "flashinfer.gdn_kernels.gdn_replay_mtp_umma" - else: - api_name = f"gated_delta_rule_{mode}_ucache_flush" - suffix = "stp" if mode == "stp" else "flush" - expected = f"flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_{suffix}" - setattr(module, api_name, kernel) - original_import = builtins.__import__ - - def import_provider(name, *args, **kwargs): - if name.startswith("flashinfer"): - assert name == expected, f"Unselected dependency: {name}" - return module - return original_import(name, *args, **kwargs) - - monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") - monkeypatch.setattr(builtins, "__import__", import_provider) - check_gdn_replayssm_dependencies.cache_clear() - try: - state_dtype = torch.float16 if mode == "umma" else torch.bfloat16 - assert check_gdn_replayssm_dependencies(width, False, state_dtype) is kernel - finally: - check_gdn_replayssm_dependencies.cache_clear() -@pytest.mark.parametrize("loaded_dtype", [torch.bfloat16, torch.float16]) -def test_materializer_rejects_dtype_change_after_import(monkeypatch, loaded_dtype): - """Changing the environment cannot change an imported provider's state dtype.""" +@pytest.mark.parametrize("loaded_dtype", [torch.float16, torch.bfloat16]) +def test_materializer_requires_fp16_state(monkeypatch, loaded_dtype): + """A FlashInfer imported before vLLM set its state dtype must be rejected.""" provider = ModuleType("provider") materialize = lambda **kwargs: None monkeypatch.setattr(provider, "gdn_prefix_materialize", materialize, raising=False) @@ -67,20 +32,15 @@ def import_provider(name, *args, **kwargs): return original_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", import_provider) - _load_gdn_replayssm_materialize.cache_clear() + load_gdn_replayssm_materializer.cache_clear() try: - assert _load_gdn_replayssm_materialize(loaded_dtype) is materialize - requested_dtype = ( - torch.float16 if loaded_dtype == torch.bfloat16 else torch.bfloat16 - ) - monkeypatch.setenv( - "GDN_UCACHE_STATE_DTYPE", - "fp16" if requested_dtype == torch.float16 else "bf16", - ) - with pytest.raises(ValueError, match="before process startup"): - _load_gdn_replayssm_materialize(requested_dtype) + if loaded_dtype == torch.float16: + assert load_gdn_replayssm_materializer() is materialize + else: + with pytest.raises(ValueError, match="GDN_UCACHE_STATE_DTYPE=fp16"): + load_gdn_replayssm_materializer() finally: - _load_gdn_replayssm_materialize.cache_clear() + load_gdn_replayssm_materializer.cache_clear() def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): @@ -112,24 +72,12 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): aligned = pack_replayssm_rows(unaligned, starts, width, uniform=True) assert aligned.data_ptr() % 16 == 0 torch.testing.assert_close(aligned, packed) - strided = torch.zeros(2 * width, 8, dtype=values.dtype)[:, ::2] - strided.copy_(values) - repacked = pack_replayssm_rows( - strided, starts, width, uniform=True, allow_strided_rows=True - ) - assert repacked.is_contiguous() - torch.testing.assert_close(repacked, packed) + # A column slice of a wider projection is copied to a contiguous buffer. projection = torch.zeros(2 * width, 8, dtype=values.dtype)[:, :4] projection.copy_(values) - viewed = pack_replayssm_rows( - projection, starts, width, uniform=True, allow_strided_rows=True - ) - assert viewed.data_ptr() == projection.data_ptr() - assert viewed.stride() == (width * 8, 8, 1) - torch.testing.assert_close(viewed, packed) - gate = pack_replayssm_rows(projection, starts, width, uniform=True) - assert gate.is_contiguous() and gate.data_ptr() % 16 == 0 - torch.testing.assert_close(gate, packed) + sliced = pack_replayssm_rows(projection, starts, width, uniform=True) + assert sliced.is_contiguous() and sliced.data_ptr() % 16 == 0 + torch.testing.assert_close(sliced, packed) # Eager staging can include a zero-token graph-padding row. padded_starts = torch.tensor([0, width, width]) padded = pack_replayssm_rows(values[:width], padded_starts, width, uniform=True) @@ -137,20 +85,6 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): assert padded[1].eq(0).all() -@pytest.mark.parametrize("query_lens", [[1, 1], [1, 0]]) -def test_single_token_packing_matches_ragged_path(query_lens): - """STP copies strided inputs and preserves zero-token padding rows.""" - num_tokens = sum(query_lens) - values = torch.arange(num_tokens * 8, dtype=torch.bfloat16).view(num_tokens, 8) - values = values[:, ::2] - starts = torch.tensor([0, *torch.tensor(query_lens).cumsum(0).tolist()]) - packed = pack_replayssm_rows(values, starts, 4, single_token=True) - expected = pack_replayssm_rows(values, starts, 4) - torch.testing.assert_close(packed, expected, rtol=0, atol=0) - assert packed.data_ptr() % 16 == 0 - assert packed[:, 1:].eq(0).all() - - def _reference_gdn_step( state, q, @@ -212,43 +146,20 @@ def _reconstruct_replayssm_state( @pytest.mark.skipif( - not torch.cuda.is_available() or torch.cuda.get_device_capability() < (9, 0), - reason="FlashInfer GDN ReplaySSM requires an SM90+ CUDA device", + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="FlashInfer GDN ReplaySSM requires an SM100-class CUDA device", ) @pytest.mark.parametrize( - "width,query_len,accepted,state_dtype", - [ - (1, 1, 1, torch.bfloat16), - (4, 3, 2, torch.bfloat16), - (4, 4, 2, torch.bfloat16), - (8, 6, 4, torch.bfloat16), - (8, 8, 4, torch.bfloat16), - (1, 1, 1, torch.float16), - (4, 3, 2, torch.float16), - (8, 6, 4, torch.float16), - ], + "width,query_len,accepted", + [(1, 1, 1), (4, 3, 2), (4, 4, 2), (8, 6, 4), (8, 8, 4)], ) @torch.inference_mode() -def test_gdn_replayssm_output_and_accepted_state( - monkeypatch, width, query_len, accepted, state_dtype -): +def test_gdn_replayssm_output_and_accepted_state(width, query_len, accepted): """Check nonzero outputs and accepted state through replay, wrap, and flush.""" - monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") - if state_dtype == torch.float16: - if torch.cuda.get_device_capability()[0] != 10: - pytest.skip("FP16 GDN ReplaySSM requires an SM100-class CUDA device") - module = "gdn_replay_mtp_umma" - else: - module = ( - "gdn_decode_bf16_wy_ucache_stp" - if width == 1 - else "gdn_decode_bf16_wy_ucache_flush" - ) - pytest.importorskip(f"flashinfer.gdn_kernels.{module}") + pytest.importorskip("flashinfer.gdn_kernels.gdn_replay_mtp_umma") device, dtype = torch.device("cuda"), torch.bfloat16 generator = torch.Generator(device=device).manual_seed(20260909 + width) - native_stp = width == 1 and state_dtype == torch.bfloat16 - ring_slots = 16 if native_stp else 32 + ring_slots = 32 def random_tensor(*shape): return torch.randn(*shape, generator=generator, device=device).to(dtype) @@ -260,7 +171,7 @@ def random_tensor(*shape): (1, ring_slots, 128), (2, ring_slots), ] - dtypes = [state_dtype, dtype, dtype, torch.float32] + dtypes = [torch.float16, dtype, dtype, torch.float32] sizes = [math.prod(shape) * dt.itemsize for shape, dt in zip(shapes, dtypes)] pages = torch.zeros(2, sum(sizes) + 16, dtype=torch.uint8, device=device) states = [] @@ -272,17 +183,17 @@ def random_tensor(*shape): pages[0].fill_(17) # Nonzero canaries catch writes to the padding page. checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) ring_start = torch.zeros(2, dtype=torch.int32, device=device) - if width == 1 and not native_stp: - ring_start[1] = 24 # Cross the UMMA ring boundary before the first flush. + if width == 1: + ring_start[1] = 24 # Cross the ring boundary before the first flush. committed = torch.zeros_like(ring_start) A_log = torch.nn.Parameter(torch.full((2,), -3.0, device=device)) dt_bias = torch.nn.Parameter(random_tensor(2) * 0.2) accepted_state = checkpoint[1].float().clone() padding_page = pages[0].clone() - capture = width == 1 or (state_dtype == torch.float16 and width == 4) + capture = width in (1, 4) graph = None - # Eighteen steps cross the native STP flush and the MTP ring boundary. + # Eighteen steps cross a flush and the ring boundary. for step in range(18): if capture and step == 9: # Captured execution must observe an in-place weight reload. @@ -300,7 +211,7 @@ def random_tensor(*shape): a, b = gates[:, 0], gates[:, 1] if capture: # Captured padding has input storage but no token or live state. - padding_tokens = 1 if width == 1 else width + padding_tokens = width mixed_qkv = torch.cat( ( mixed_qkv, @@ -334,7 +245,8 @@ def random_tensor(*shape): dtype=torch.int32, device=device, ), - executed_query_width=width, + kernel=load_gdn_replayssm_kernel(), + executed_width=width, uniform_query_length=query_len, num_k_heads=1, num_v_heads=2, @@ -380,9 +292,7 @@ def random_tensor(*shape): dt_bias, ) old_committed = int(committed[1].item()) - if native_stp: - committed[1] = 0 if old_committed >= 15 else old_committed + 1 - elif old_committed + width > 16: + if old_committed + width > 16: ring_start[1] = (ring_start[1] + old_committed) % ring_slots committed[1] = accepted else: diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index 2472d5e752c8..a902cecd21e4 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -4,6 +4,7 @@ import os from dataclasses import replace from types import SimpleNamespace +from typing import NamedTuple from unittest.mock import Mock import pytest @@ -314,26 +315,45 @@ def test_replayssm_materializes_only_accepted_boundary_with_compacted_rows( torch.testing.assert_close(states[1], before_shrink[name], atol=0, rtol=0) +class _PostprocessCase(NamedTuple): + ring_slots: int + executed_width: int # 0: fold decisions use the row's query length. + query_len: int + committed: int # Live slot history before the step. + flush_count: int + ring_start: int + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("post_step", [False, True], ids=["v1", "v2"]) -@pytest.mark.parametrize("gdn_mode", ["none", "stp", "umma_stp"]) +@pytest.mark.parametrize( + "case", + [ + # Mamba2: 2 + 4 fits the window, so replay 2 old + 4 accepted tokens. + pytest.param(_PostprocessCase(20, 0, 4, 2, 6, 3), id="mamba2"), + # GDN standard decoding: 16 + 1 > 16 folds; publish only the new token. + pytest.param(_PostprocessCase(32, 1, 1, 16, 1, 19), id="gdn-stp-fold"), + # A 4-token row in a width-8 GDN launch folds at 10 + 8 > 16, although + # 10 + 4 would fit. + pytest.param(_PostprocessCase(32, 8, 4, 10, 4, 13), id="gdn-mtp-fold"), + pytest.param(_PostprocessCase(32, 8, 4, 8, 12, 3), id="gdn-mtp-append"), + ], +) def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( - post_step, gdn_mode + post_step, case ): """Skip prefill self-copies, reset trackers, and preserve in-place MTP replay.""" - gdn_stp = gdn_mode == "stp" - umma_stp = gdn_mode == "umma_stp" - ring_slots = 16 if gdn_stp else 32 if umma_stp else 20 + slots = case.ring_slots mixer = _replayssm_mixer( kv_cache=( torch.empty(0, device="cuda"), torch.empty(6, 1, 1, 1, device="cuda"), - torch.empty(6, 1, ring_slots, 1, device="cuda"), - torch.empty(6, 1, ring_slots, device="cuda"), - torch.empty(6, 1, ring_slots, 1, device="cuda"), + torch.empty(6, 1, slots, 1, device="cuda"), + torch.empty(6, 1, slots, device="cuda"), + torch.empty(6, 1, slots, 1, device="cuda"), ), ring_start=_cuda_ints([3] * 6), - committed=_cuda_ints([2, 2, 2, 2, 15 if gdn_stp else 16 if umma_stp else 2, 2]), + committed=_cuda_ints([2, 2, 2, 2, case.committed, 2]), A=torch.empty(1, device="cuda"), ) spec = MambaSpec( @@ -351,9 +371,10 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( ] ) context = ReplaySSMModelContext.create([([mixer], spec, table)], max_num_reqs=4) - # Reordered rows: canonical prefill in slot 2; in-place MTP replay in slot 4; - # padded slot; padded request. - query_len = 1 if gdn_stp or umma_stp else 4 + # Reordered rows: canonical prefill in slot 2; in-place replay in slot 4 + # that accepts every token and reaches the block boundary at 32; padded + # slot; padded request. + query_len = case.query_len _postprocess_replayssm_kernel[(4, 1)]( _cuda_ints([1, 2, 0, -1]), _cuda_ints([0, 48, 48 + query_len, 64 + query_len, 64 + query_len]) @@ -375,15 +396,14 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( MAX_NUM_REQS=4, MAMBA_BLOCK_SIZE=16, LOGICAL_WINDOW=16, - RING_BUFFER_LEN=ring_slots, + RING_BUFFER_LEN=slots, PAD_SLOT_ID=NULL_BLOCK_ID, QUERY_METADATA_IS_CUMULATIVE=post_step, NUM_COMPUTED_IS_POST_STEP=post_step, HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, - EXECUTED_QUERY_WIDTH=1 if gdn_stp or umma_stp else 0, - GDN_STP=gdn_stp, + EXECUTED_QUERY_WIDTH=case.executed_width, ) _compact_replayssm_requests_kernel[(1,)]( context.plan_flush_count, @@ -394,12 +414,8 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( BLOCK_SIZE=4, ) - assert context.plan_flush_count.tolist() == [ - -1, - 0 if gdn_stp else 1 if umma_stp else 6, - -1, - -1, - ] + assert context.plan_flush_count.tolist() == [-1, case.flush_count, -1, -1] + assert context.plan_ring_start[1].item() == case.ring_start assert context.active_request_indices.tolist() == [1, -1, -1, -1] assert context.src_slots[0, 1].item() == 4 assert context.dst_slots[0, 1].item() == 4 diff --git a/tests/test_config.py b/tests/test_config.py index 3c030705a7e1..c56284017369 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -158,6 +158,51 @@ def test_flashinfer_replayssm_validates_config( VllmConfig.validate_mamba_cached_kernel(config) +def _gdn_replayssm_config(mode: str, ssm_dtype: str = "float16") -> SimpleNamespace: + return SimpleNamespace( + cache_config=SimpleNamespace( + use_replayssm=True, + use_kda_recoverssm=False, + replayssm_buffer_len=16, + mamba_cache_mode=mode, + enable_prefix_caching=mode == "align", + mamba_cache_dtype="auto", + mamba_ssm_cache_dtype=ssm_dtype, + ), + num_speculative_tokens=3, + speculative_config=None, + model_config=SimpleNamespace( + architecture="Qwen3_5MoeForCausalLM", + dtype=torch.bfloat16, + hf_text_config=SimpleNamespace( + linear_key_head_dim=128, linear_value_head_dim=128 + ), + ), + mamba_config=SimpleNamespace(enable_stochastic_rounding=False), + parallel_config=SimpleNamespace(pipeline_parallel_size=1, use_ubatching=False), + kv_transfer_config=None, + ) + + +def test_gdn_replayssm_requires_fp16_state(): + with pytest.raises(ValueError, match="--mamba-ssm-cache-dtype float16"): + VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("none", "bfloat16")) + + +def test_gdn_replayssm_align_configures_flashinfer_materializer(monkeypatch): + """--use-replayssm selects FlashInfer's FP16 materializer before import.""" + # Register the variable so monkeypatch removes what the validator sets. + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "") + monkeypatch.delenv("GDN_UCACHE_STATE_DTYPE") + VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("none")) + assert "GDN_UCACHE_STATE_DTYPE" not in os.environ + VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("align")) + assert os.environ["GDN_UCACHE_STATE_DTYPE"] == "fp16" + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "bf16") + with pytest.raises(ValueError, match="GDN_UCACHE_STATE_DTYPE='bf16'"): + VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("align")) + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") diff --git a/tests/test_envs.py b/tests/test_envs.py index 948551a9a5f8..4298078278a8 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -300,15 +300,9 @@ def test_gdn_decode_kernel_env(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("VLLM_GDN_DECODE_KERNEL", raising=False) assert env_func() == "cuda" - for value in ( - "cuda", - "triton", - "flashinfer_replayssm", - "CUDA", - "FLASHINFER_REPLAYSSM", - ): + for value in ("cuda", "triton"): monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) - assert env_func() == value.lower() + assert env_func() == value for value in ("fused", "invalid"): monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", value) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index ae49cd60eb0b..fd9842073af6 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -285,7 +285,7 @@ def test_has_initial_state_after_reclassification(): assert meta.has_initial_state[0].item() is True -def test_replayssm_captured_indices_remain_valid_after_ragged_batch(monkeypatch): +def test_replayssm_captured_indices_remain_valid_after_ragged_batch(): """A smaller FULL graph must not read stale indices from a ragged step.""" config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE) config.cache_config.use_replayssm = True @@ -293,7 +293,6 @@ def test_replayssm_captured_indices_remain_valid_after_ragged_batch(monkeypatch) config.speculative_config = SpeculativeConfig( method="ngram", num_speculative_tokens=3 ) - monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", "flashinfer_replayssm") builder = GDNAttentionMetadataBuilder( kv_cache_spec=MambaSpec( block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.bfloat16,) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index f13d1980496b..f86e974a650a 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -2028,12 +2028,10 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks(): runner.initialize_kv_cache(kv_cache_config) -def test_gdn_intermediate_prefill_rejects_full_graph_dispatch(dist_init, monkeypatch): +def test_gdn_intermediate_prefill_rejects_full_graph_dispatch(dist_init): from vllm.config import CompilationConfig, CUDAGraphMode from vllm.v1.worker.gpu_input_batch import CachedRequestState - monkeypatch.setenv("VLLM_GDN_DECODE_KERNEL", "flashinfer_replayssm") - monkeypatch.setenv("SGLANG_GDN_WY_STRIDED_QKV", "1") config = VllmConfig( model_config=ModelConfig(model="Qwen/Qwen3.5-0.8B", dtype="bfloat16"), scheduler_config=SchedulerConfig( @@ -2046,7 +2044,7 @@ def test_gdn_intermediate_prefill_rejects_full_graph_dispatch(dist_init, monkeyp use_replayssm=True, replayssm_buffer_len=16, mamba_cache_dtype="bfloat16", - mamba_ssm_cache_dtype="bfloat16", + mamba_ssm_cache_dtype="float16", mamba_cache_mode="none", enable_prefix_caching=False, ), diff --git a/vllm/config/cache.py b/vllm/config/cache.py index f0a88c40f024..ce0b5f2e8f8f 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -320,10 +320,9 @@ def device_memory_utilization(self, value: float | None) -> None: block size.""" replayssm_buffer_len: int = Field(default=16, gt=0) """ReplaySSM logical history length B. Mamba2 Triton uses B physical rows - and Mamba2 FlashInfer uses B+T, where T is the verification length. GDN - FlashInfer requires B=16 and uses a 16-row physical ring for native STP or - a 32-row ring for MTP. Kimi-K3 speculative decode does not use B. Default - 16.""" + and Mamba2 FlashInfer uses B+T, where T is the verification length. Qwen3.5 + Gated DeltaNet requires B=16 and uses 32 physical rows. Kimi-K3 speculative + decode does not use B. Default 16.""" use_replayssm: bool = False """Use a supported ReplaySSM decode kernel: cache recent recurrent inputs and skip the per-step full-state store, writing the checkpoint only on flush. diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2a19dbdced24..932ab453877c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -141,6 +141,16 @@ ) +def _uses_gdn_replayssm( + cache_config: CacheConfig, model_config: "ModelConfig | None" +) -> bool: + return bool( + cache_config.use_replayssm + and model_config is not None + and model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES + ) + + @lru_cache def default_breakable_cudagraph_architectures() -> frozenset[str]: """Architectures defaulting to breakable CUDA graphs on this platform.""" @@ -650,8 +660,9 @@ def max_in_flight_tokens(self) -> int: @property def use_flashinfer_replayssm(self) -> bool: + """Whether a FlashInfer ReplaySSM kernel runs Mamba2 or GDN decode.""" return self.cache_config.use_replayssm and ( - VllmConfig.is_gdn_replayssm_enabled(self) + _uses_gdn_replayssm(self.cache_config, self.model_config) or self.mamba_config.backend == MambaBackendEnum.FLASHINFER ) @@ -3946,142 +3957,85 @@ def validate_mamba_block_size(self) -> "VllmConfig": @model_validator(mode="after") def validate_gdn_replayssm(self) -> "VllmConfig": - # Call the implementation on the class so this validator remains usable - # by focused tests that pass a config-shaped object rather than a full - # pydantic model. - use_gdn_replayssm = VllmConfig.is_gdn_replayssm_enabled(self) - is_gdn_replayssm_architecture = ( - self.model_config is not None - and self.model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES - ) + if not _uses_gdn_replayssm(self.cache_config, self.model_config): + return self + cache_config = self.cache_config + if not 0 <= self.num_speculative_tokens <= 7: + raise ValueError( + "FlashInfer GDN ReplaySSM supports at most 7 speculative tokens" + ) if ( - envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" - and not self.cache_config.use_replayssm + self.speculative_config is not None + and self.speculative_config.enable_adaptive_verification ): raise ValueError( - "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm requires --use-replayssm" + "FlashInfer GDN ReplaySSM does not support adaptive verification; " + "set enable_adaptive_verification=False" + ) + if cache_config.replayssm_buffer_len != 16: + raise ValueError( + "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" + ) + if cache_config.mamba_cache_mode not in ("none", "align"): + raise ValueError( + "FlashInfer GDN ReplaySSM supports only the none and align " + "Mamba cache modes" ) if ( - envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" - and self.model_config is not None - and not use_gdn_replayssm + cache_config.enable_prefix_caching + and cache_config.mamba_cache_mode != "align" ): raise ValueError( - "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm currently supports " - "only Qwen3.5 architectures" + "FlashInfer GDN ReplaySSM prefix caching requires " + "--mamba-cache-mode=align" ) + if self.model_config.dtype != torch.bfloat16: + raise ValueError("FlashInfer GDN ReplaySSM requires bfloat16 model dtype") + hf_config = self.model_config.hf_text_config if ( - self.cache_config.use_replayssm - and is_gdn_replayssm_architecture - and envs.VLLM_GDN_DECODE_KERNEL != "flashinfer_replayssm" + getattr(hf_config, "linear_key_head_dim", None) != 128 + or getattr(hf_config, "linear_value_head_dim", None) != 128 ): + raise ValueError("FlashInfer GDN ReplaySSM requires GDN K=V=128") + if cache_config.mamba_cache_dtype not in ("auto", "bfloat16"): raise ValueError( - "--use-replayssm on Qwen3.5 requires " - "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm" + "FlashInfer GDN ReplaySSM requires a bfloat16 convolution cache" ) - if not self.cache_config.use_replayssm: - self.cache_config.use_kda_recoverssm = False - return self - - if use_gdn_replayssm: + if cache_config.mamba_ssm_cache_dtype != "float16": + raise ValueError( + "FlashInfer GDN ReplaySSM requires --mamba-ssm-cache-dtype float16" + ) + if self.mamba_config.enable_stochastic_rounding: + raise ValueError( + "FlashInfer GDN ReplaySSM does not support " + "--enable-mamba-cache-stochastic-rounding" + ) + # Replay rings and cursors are not part of the pipeline, microbatch, or + # KV-connector state handoff. + if self.parallel_config.pipeline_parallel_size > 1: + raise ValueError( + "FlashInfer GDN ReplaySSM currently requires pipeline_parallel_size=1" + ) + if self.parallel_config.use_ubatching: + raise ValueError( + "FlashInfer GDN ReplaySSM does not support overlapping " + "microbatch execution" + ) + if ( + self.kv_transfer_config is not None + and self.kv_transfer_config.is_kv_transfer_instance + ): + raise ValueError( + "--use-replayssm is incompatible with KV connectors " + "(P/D disaggregation, KV cache offload)" + ) + if cache_config.mamba_cache_mode == "align": from vllm.model_executor.layers.mamba.mamba_utils import ( - gdn_replayssm_geometry, + configure_flashinfer_gdn_env, ) - # Integration constraints: fixed verify widths and none/align caches. - if ( - self.speculative_config is not None - and self.speculative_config.enable_adaptive_verification - ): - raise ValueError( - "FlashInfer GDN ReplaySSM does not support adaptive verification; " - "set enable_adaptive_verification=False" - ) - if self.cache_config.mamba_cache_mode not in ("none", "align"): - raise ValueError( - "FlashInfer GDN ReplaySSM supports only none and align " - "Mamba cache modes; Qwen3.5 does not support all mode" - ) - if ( - self.cache_config.enable_prefix_caching - and self.cache_config.mamba_cache_mode != "align" - ): - raise ValueError( - "FlashInfer GDN ReplaySSM prefix caching requires " - "--mamba-cache-mode=align" - ) - # Selected provider constraints: history tile and launch widths. - if self.cache_config.replayssm_buffer_len != 16: - raise ValueError( - "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" - ) - gdn_replayssm_geometry(self.num_speculative_tokens) - # Both provider paths require BF16 activations and convolution state. - if self.model_config is not None: - if self.model_config.dtype != torch.bfloat16: - raise ValueError( - "FlashInfer GDN ReplaySSM requires bfloat16 model dtype" - ) - hf_config = self.model_config.hf_text_config - if ( - getattr(hf_config, "linear_key_head_dim", None) != 128 - or getattr(hf_config, "linear_value_head_dim", None) != 128 - ): - raise ValueError("FlashInfer GDN ReplaySSM requires GDN K=V=128") - if self.cache_config.mamba_cache_dtype not in ("auto", "bfloat16"): - raise ValueError( - "FlashInfer GDN ReplaySSM requires a bfloat16 convolution cache" - ) - if self.cache_config.mamba_ssm_cache_dtype not in ( - "auto", - "bfloat16", - "float16", - ): - raise ValueError( - "FlashInfer GDN ReplaySSM requires BF16 or FP16 recurrent state" - ) - if self.cache_config.mamba_ssm_cache_dtype == "float16": - if self.mamba_config.enable_stochastic_rounding: - raise ValueError( - "FP16 GDN ReplaySSM does not support " - "--enable-mamba-cache-stochastic-rounding" - ) - if ( - self.cache_config.mamba_cache_mode == "align" - and os.environ.get("GDN_UCACHE_STATE_DTYPE") != "fp16" - ): - raise ValueError( - "FP16 GDN ReplaySSM prefix materialization requires " - "GDN_UCACHE_STATE_DTYPE=fp16 before process startup" - ) - # These execution modes need additional tracker lifecycle support. - if self.parallel_config.pipeline_parallel_size > 1: - raise ValueError( - "FlashInfer GDN ReplaySSM currently requires " - "pipeline_parallel_size=1" - ) - if self.parallel_config.use_ubatching: - raise ValueError( - "FlashInfer GDN ReplaySSM does not support overlapping " - "microbatch execution" - ) - # The provider selects its strided-QKV specialization at import time. - if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": - raise ValueError( - "FlashInfer GDN ReplaySSM requires " - "SGLANG_GDN_WY_STRIDED_QKV=1 before process startup" - ) - # This integration does not transfer replay rings and trackers. - if ( - self.kv_transfer_config is not None - and self.kv_transfer_config.is_kv_transfer_instance - ): - raise ValueError( - "--use-replayssm is incompatible with KV connectors " - "(P/D disaggregation, KV cache offload)" - ) - self.cache_config.use_kda_recoverssm = False - return self + configure_flashinfer_gdn_env() + cache_config.use_kda_recoverssm = False return self @model_validator(mode="after") @@ -4089,8 +4043,8 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": if not self.cache_config.use_replayssm: self.cache_config.use_kda_recoverssm = False return self - # The preceding GDN validator owns its provider and lifecycle checks. - if VllmConfig.is_gdn_replayssm_enabled(self): + # validate_gdn_replayssm owns the GDN checks. + if _uses_gdn_replayssm(self.cache_config, self.model_config): return self kda_architectures = ( @@ -4189,13 +4143,8 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": return self def is_gdn_replayssm_enabled(self) -> bool: - """Whether this config selects the FlashInfer GDN replay adapter.""" - return bool( - self.cache_config.use_replayssm - and envs.VLLM_GDN_DECODE_KERNEL == "flashinfer_replayssm" - and self.model_config is not None - and self.model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES - ) + """Whether --use-replayssm selects FlashInfer GDN ReplaySSM.""" + return _uses_gdn_replayssm(self.cache_config, self.model_config) _current_vllm_config: VllmConfig | None = None diff --git a/vllm/envs.py b/vllm/envs.py index 426cb4baf5db..20de13f59da8 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -9,7 +9,7 @@ import tempfile import uuid from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: VLLM_HOST_IP: str = "" @@ -133,7 +133,7 @@ VLLM_DISABLED_KERNELS: list[str] = [] VLLM_USE_HW_AGNOSTIC: bool = False VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE: bool = True - VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton", "flashinfer_replayssm"] = "cuda" + VLLM_GDN_DECODE_KERNEL: Literal["cuda", "triton"] = "cuda" VLLM_DISABLE_PYNCCL: bool = False VLLM_USE_OINK_OPS: bool = False VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD: bool = False @@ -1281,19 +1281,15 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": lambda: bool( int(os.getenv("VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE", "1")) ), - # Select the GDN decode implementation. "cuda" uses the fused decode kernel - # where supported and falls back to "triton" otherwise; explicitly selecting - # "cuda" raises when unsupported. "flashinfer_replayssm" selects checkpoint - # and replay execution and requires --use-replayssm. - "VLLM_GDN_DECODE_KERNEL": lambda: cast( - str, - env_with_choices( - "VLLM_GDN_DECODE_KERNEL", - "cuda", - ["cuda", "triton", "flashinfer_replayssm"], - case_sensitive=False, - )(), - ).lower(), + # Select the GDN MTP decode implementation. "cuda" uses the fused decode + # kernel where supported and falls back to "triton" otherwise; setting it + # explicitly to "cuda" raises when unsupported. + "VLLM_GDN_DECODE_KERNEL": env_with_choices( + "VLLM_GDN_DECODE_KERNEL", + "cuda", + ["cuda", "triton"], + case_sensitive=False, + ), # Disable pynccl (using torch.distributed instead) "VLLM_DISABLE_PYNCCL": lambda: ( os.getenv("VLLM_DISABLE_PYNCCL", "False").lower() in ("true", "1") diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index ddd7fdc32775..2910eb59abff 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -14,7 +14,7 @@ from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, - gdn_replayssm_geometry, + gdn_replayssm_executed_width, mamba_state_alignment, ) from vllm.model_executor.models.utils import extract_layer_index @@ -48,28 +48,18 @@ def __init__( else 0 ) self.use_replayssm = vllm_config.is_gdn_replayssm_enabled() + # GDN ReplaySSM always runs on FlashInfer; MambaBase consumers read this. self.use_flashinfer_replayssm = self.use_replayssm self.state_alignment = mamba_state_alignment(vllm_config) self.replayssm_buffer_len = ( self.cache_config.replayssm_buffer_len if self.use_replayssm else None ) - self.replayssm_executed_query_width: int | None = None - self.replayssm_ring_slots: int | None = None + self.replayssm_executed_width: int | None = None if self.use_replayssm: - self.replayssm_executed_query_width, self.replayssm_ring_slots = ( - gdn_replayssm_geometry(self.num_spec, self.get_state_dtype()[1]) - ) - if self.replayssm_executed_query_width is not None: - # BF16 STP accepts one real token and three zero input positions. - # These offsets pack token inputs; they are not replay-ring cursors. - packed_width = ( - 4 - if self.replayssm_ring_slots == 16 - else self.replayssm_executed_query_width - ) + self.replayssm_executed_width = gdn_replayssm_executed_width(self.num_spec) self.register_buffer( "_replayssm_offsets", - torch.arange(packed_width, dtype=torch.int32), + torch.arange(self.replayssm_executed_width, dtype=torch.int32), persistent=False, ) self._replayssm_ring_start = torch.empty(0, dtype=torch.int32) @@ -79,6 +69,12 @@ def __init__( def mamba_type(self) -> MambaAttentionBackendEnum: return MambaAttentionBackendEnum.GDN_ATTN + @property + def replayssm_rings(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Bound ``(u, k, G)`` rings, stored after the conv and SSM states.""" + u_cache, k_cache, g_cache = self.kv_cache[2:5] + return u_cache, k_cache, g_cache + def get_state_dtype(self) -> tuple[torch.dtype, ...]: dtypes: tuple[torch.dtype, ...] = ( MambaStateDtypeCalculator.gated_delta_net_state_dtype( diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 1da7a95be1c4..6a6ccde816b9 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -45,7 +45,8 @@ ) from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention from vllm.model_executor.layers.mamba.gdn.replayssm import ( - check_gdn_replayssm_dependencies, + load_gdn_replayssm_kernel, + load_gdn_replayssm_materializer, run_gdn_replayssm, ) from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader @@ -693,7 +694,6 @@ def get_state_shape( ) if self.use_replayssm: - assert self.replayssm_ring_slots is not None shapes = ( *shapes, *MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( @@ -702,7 +702,6 @@ def get_state_shape( self.num_v_heads, self.head_k_dim, self.head_v_dim, - ring_slots=self.replayssm_ring_slots, ), ) return shapes @@ -880,18 +879,12 @@ def __init__( self._fused_gdn_gated_rms_norm = fused_gdn_gated_rms_norm self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower() if self.use_replayssm: - reason = self._flashinfer_replayssm_unsupported_reason(vllm_config) + reason = self._flashinfer_replayssm_unsupported_reason() if reason is not None: - raise ValueError( - "VLLM_GDN_DECODE_KERNEL=flashinfer_replayssm is not " - f"supported: {reason}" - ) - assert self.replayssm_executed_query_width is not None - check_gdn_replayssm_dependencies( - self.replayssm_executed_query_width, - needs_materializer=self.cache_config.mamba_cache_mode == "align", - state_dtype=self.get_state_dtype()[1], - ) + raise ValueError(f"--use-replayssm is not supported: {reason}") + self._replayssm_kernel = load_gdn_replayssm_kernel() + if self.cache_config.mamba_cache_mode == "align": + load_gdn_replayssm_materializer() elif self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): reason = self._fused_gdn_decode_unsupported_reason(vllm_config) if reason is not None: @@ -921,33 +914,17 @@ def __init__( raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self - def _flashinfer_replayssm_unsupported_reason( - self, vllm_config: VllmConfig - ) -> str | None: - conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2] + def _flashinfer_replayssm_unsupported_reason(self) -> str | None: + # validate_gdn_replayssm checks dtypes and head sizes from the config. + if not current_platform.is_device_capability_family(100): + return "FlashInfer's UMMA kernel requires an SM100-class GPU" if self.gqa_interleaved_layout: - return "interleaved GQA layout is not supported by this integration" - if self.head_k_dim != 128 or self.head_v_dim != 128: - return "the selected FlashInfer kernel requires K=V=128" + return "the interleaved GQA layout is not supported" if self.num_k_heads % self.tp_size or self.num_v_heads % self.tp_size: return "tensor-parallel head counts must divide evenly" - if self.num_v_heads % self.num_k_heads: - return "value heads must be an integer multiple of key heads" - if vllm_config.model_config.dtype != torch.bfloat16: - return "the GDN ReplaySSM adapter requires bfloat16 model activations" - if conv_state_dtype != torch.bfloat16: - return "the GDN ReplaySSM adapter requires bfloat16 convolution state" - if recurrent_state_dtype == torch.float16: - if not current_platform.is_device_capability_family(100): - return "FP16 GDN ReplaySSM requires an SM100-class CUDA device" - if self.num_v_heads // self.num_k_heads > 4: - return ( - "FP16 GDN ReplaySSM supports at most four value heads per key head" - ) - elif recurrent_state_dtype != torch.bfloat16: - return "the GDN ReplaySSM adapter requires BF16 or FP16 recurrent state" - if not current_platform.has_device_capability(90): - return "the initial integration requires an SM90+ CUDA device" + heads_per_key, remainder = divmod(self.num_v_heads, self.num_k_heads) + if remainder or heads_per_key > 4: + return "FlashInfer's UMMA kernel needs 1 to 4 value heads per key head" return None def get_prefill_checkpoint_alignment(self) -> int | None: @@ -1924,8 +1901,7 @@ def _forward_core( num_accepted_tokens=num_accepted_tokens, query_start_loc=spec_query_start_loc, max_query_len=( - attn_metadata.replayssm_executed_query_width - or spec_state_indices_tensor.size(-1) + self.replayssm_executed_width or spec_state_indices_tensor.size(-1) ), validate_data=False, ) @@ -1965,8 +1941,7 @@ def _forward_core( mixed_qkv_non_spec = None if self.use_replayssm and spec_sequence_masks is not None: - # ReplaySSM splits packed QKV inside its adapter. Other decode - # paths consume the separate query, key, and value views below. + # run_gdn_replayssm splits the packed QKV itself. query_spec, key_spec, value_spec = None, None, None else: query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) @@ -2038,23 +2013,24 @@ def _forward_core( assert spec_state_indices_tensor is not None assert spec_query_start_loc is not None assert attn_metadata.replayssm_output_indices is not None - assert attn_metadata.replayssm_executed_query_width is not None + assert self.replayssm_executed_width is not None num_replay_rows = attn_metadata.num_spec_decodes core_attn_out_spec = run_gdn_replayssm( + kernel=self._replayssm_kernel, + executed_width=self.replayssm_executed_width, mixed_qkv=mixed_qkv_spec, a=a_spec, b=b_spec, A_log=self.A_log, dt_bias=self.dt_bias, checkpoint_state=ssm_state, - replayssm_cache=self.kv_cache[2:5], + replayssm_cache=self.replayssm_rings, ring_start=self._replayssm_ring_start, num_committed=self._replayssm_prev_num_accepted, query_start_loc=spec_query_start_loc[: num_replay_rows + 1], state_indices=spec_state_indices_tensor[:num_replay_rows, 0], output_indices=attn_metadata.replayssm_output_indices, uniform_query_length=attn_metadata.uniform_spec_sequence_length, - executed_query_width=(attn_metadata.replayssm_executed_query_width), num_k_heads=self.num_k_heads // self.tp_size, num_v_heads=self.num_v_heads // self.tp_size, head_k_dim=self.head_k_dim, @@ -2205,8 +2181,8 @@ def _forward_core( ) elif spec_sequence_masks is not None: if self.use_replayssm: - # FULL graphs may include padded token lanes after all real - # replay rows. Scatter only the gathered real-token outputs. + # FULL graphs pad tokens after the real replay rows; scatter + # only the real-token outputs. assert spec_token_indx is not None core_attn_out.index_copy_( 0, spec_token_indx.long(), core_attn_out_spec.squeeze(0) diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index f20fdb1b96cf..a9902d3e9aaf 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -1,270 +1,176 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Adapters for FlashInfer's GDN ReplaySSM decode kernels.""" +"""Adapter for FlashInfer's FP16-state GDN ReplaySSM decode kernel.""" -import os from collections.abc import Callable from functools import cache import torch +from vllm.model_executor.layers.mamba.mamba_utils import GDN_REPLAYSSM_LOGICAL_WINDOW from vllm.v1.attention.backends.utils import NULL_BLOCK_ID -GDN_REPLAY_LOGICAL_WINDOW = 16 +# CuTe loads every kernel input with 16-byte vector accesses. +_CUTE_ALIGNMENT_BYTES = 16 @cache -def check_gdn_replayssm_dependencies( - executed_width: int, - needs_materializer: bool, - state_dtype: torch.dtype = torch.bfloat16, -) -> Callable[..., torch.Tensor]: - """Validate the environment and return only the selected decode API.""" - if executed_width not in (1, 4, 8): - raise ValueError(f"Unsupported GDN ReplaySSM executed width: {executed_width}") - if state_dtype not in (torch.bfloat16, torch.float16): - raise ValueError(f"Unsupported GDN ReplaySSM state dtype: {state_dtype}") - use_umma = state_dtype == torch.float16 - if os.environ.get("SGLANG_GDN_WY_STRIDED_QKV") != "1": - raise RuntimeError( - "FlashInfer GDN ReplaySSM requires " - "SGLANG_GDN_WY_STRIDED_QKV=1 before importing FlashInfer" - ) - if use_umma: - api = "gated_delta_rule_mtp_ucache_flush_umma" - else: - api = ( - "gated_delta_rule_stp_ucache_flush" - if executed_width == 1 - else "gated_delta_rule_mtp_ucache_flush" +def load_gdn_replayssm_kernel() -> Callable[..., None]: + """Import FlashInfer's SM100 UMMA ReplaySSM decode kernel.""" + try: + from flashinfer.gdn_kernels.gdn_replay_mtp_umma import ( + gated_delta_rule_mtp_ucache_flush_umma, ) - mode = f"executed width={executed_width}, cache mode=" + ( - "align" if needs_materializer else "none" - ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM requires " + "gated_delta_rule_mtp_ucache_flush_umma (FlashInfer >= 0.7.3)" + ) from e + return gated_delta_rule_mtp_ucache_flush_umma + + +@cache +def load_gdn_replayssm_materializer() -> Callable[..., None]: + """Import FlashInfer's prefix materializer and check its FP16 state dtype.""" try: - # MTP-only providers need not ship the separate STP API. - if use_umma: - from flashinfer.gdn_kernels.gdn_replay_mtp_umma import ( - gated_delta_rule_mtp_ucache_flush_umma as kernel, - ) - elif executed_width == 1: - from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_stp import ( - gated_delta_rule_stp_ucache_flush as kernel, - ) - else: - from flashinfer.gdn_kernels.gdn_decode_bf16_wy_ucache_flush import ( - gated_delta_rule_mtp_ucache_flush as kernel, - ) + from flashinfer.gdn_kernels.gdn_prefix_materialize import ( + gdn_prefix_materialize, + state_ty_torch, + ) except ImportError as e: raise ImportError( - f"FlashInfer GDN ReplaySSM ({mode}) requires {api}: {e}" + "FlashInfer GDN ReplaySSM prefix caching requires " + "flashinfer.gdn_kernels.gdn_prefix_materialize (FlashInfer >= 0.7.3)" ) from e - if needs_materializer: - from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( - _load_gdn_replayssm_materialize, + if state_ty_torch() != torch.float16: + raise ValueError( + f"FlashInfer GDN prefix materialization uses {state_ty_torch()} " + "state because FlashInfer was imported before vLLM configured it. " + "Set GDN_UCACHE_STATE_DTYPE=fp16 before starting the process." ) - - try: - _load_gdn_replayssm_materialize(state_dtype) - except ImportError as e: - raise ImportError( - f"FlashInfer GDN ReplaySSM ({mode}) requires " - f"gdn_prefix_materialize: {e}" - ) from e - return kernel + return gdn_prefix_materialize def pack_replayssm_rows( values: torch.Tensor, query_start_loc: torch.Tensor, - executed_query_width: int, + executed_width: int, offsets: torch.Tensor | None = None, padding_value: float = 0.0, uniform: bool = False, - single_token: bool = False, - allow_strided_rows: bool = False, ) -> torch.Tensor: - """Pack ragged rows; single_token requires at most one token per row.""" + """Pack variable-length token rows into ``[num_rows, executed_width, ...]``. + + Row ``i`` holds the tokens in ``query_start_loc[i]:query_start_loc[i + 1]`` + followed by ``padding_value``. With ``uniform``, every row has exactly + ``executed_width`` tokens, so the result is a view of ``values`` when it + is contiguous and aligned for CuTe, and a contiguous copy otherwise. + """ num_rows = query_start_loc.numel() - 1 - if single_token and padding_value == 0.0 and values.shape[0] == num_rows: - packed = values.new_zeros((num_rows, executed_query_width, *values.shape[1:])) - packed[:, 0].copy_(values) - return packed - if uniform and values.shape[0] == num_rows * executed_query_width: - if not ( - values.data_ptr() % 16 == 0 - and ( - values.is_contiguous() - or ( - allow_strided_rows - and values.ndim == 2 - and values.stride(1) == 1 - and values.stride(0) >= values.shape[1] - and values.stride(0) * values.element_size() % 16 == 0 - ) - ) - ): + if uniform and values.shape[0] == num_rows * executed_width: + if values.data_ptr() % _CUTE_ALIGNMENT_BYTES or not values.is_contiguous(): values = values.clone(memory_format=torch.contiguous_format) - return values.view(num_rows, executed_query_width, *values.shape[1:]) + return values.view(num_rows, executed_width, *values.shape[1:]) if num_rows == 0: - return values.new_empty((0, executed_query_width, *values.shape[1:])) + return values.new_empty((0, executed_width, *values.shape[1:])) if values.shape[0] == 0: raise ValueError("cannot pack non-empty ReplaySSM rows from an empty tensor") if offsets is None: offsets = torch.arange( - executed_query_width, - dtype=query_start_loc.dtype, - device=query_start_loc.device, + executed_width, dtype=query_start_loc.dtype, device=query_start_loc.device ) else: - offsets = offsets[:executed_query_width] + offsets = offsets[:executed_width] row_offsets = offsets.unsqueeze(0).expand(num_rows, -1) query_lens = torch.diff(query_start_loc).unsqueeze(1) valid = row_offsets < query_lens source_indices = query_start_loc[:-1].unsqueeze(1) + row_offsets - source_indices = source_indices.clamp(max=max(values.shape[0] - 1, 0)) + source_indices = source_indices.clamp(max=values.shape[0] - 1) packed = values.index_select(0, source_indices.reshape(-1).long()).view( - num_rows, executed_query_width, *values.shape[1:] + num_rows, executed_width, *values.shape[1:] ) - mask = valid.reshape(num_rows, executed_query_width, *([1] * (values.dim() - 1))) + mask = valid.reshape(num_rows, executed_width, *([1] * (values.dim() - 1))) return packed.masked_fill(~mask, padding_value) def run_gdn_replayssm( *, + kernel: Callable[..., None], + executed_width: int, mixed_qkv: torch.Tensor, a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor, dt_bias: torch.Tensor, checkpoint_state: torch.Tensor, - replayssm_cache: tuple[torch.Tensor, ...], + replayssm_cache: tuple[torch.Tensor, torch.Tensor, torch.Tensor], ring_start: torch.Tensor, num_committed: torch.Tensor, query_start_loc: torch.Tensor, state_indices: torch.Tensor, output_indices: torch.Tensor, - executed_query_width: int, num_k_heads: int, num_v_heads: int, head_k_dim: int, head_v_dim: int, offsets: torch.Tensor | None = None, - kernel: Callable[..., torch.Tensor] | None = None, uniform_query_length: int | None = None, ) -> torch.Tensor: - """Pack rows, invoke FlashInfer, and return only real-token outputs. + """Run one FlashInfer decode step over the replay rows. - BF16 STP receives one real token followed by three zero input positions. - FP16 UMMA receives the executed width, including one for standard decoding. - vLLM publishes cursors once per model step, after every layer runs. + Returns the outputs of real tokens in ``output_indices`` order. The kernel + reads the committed cursors but does not publish them; vLLM commits the + accepted tokens once per model step, after every layer runs. """ - if executed_query_width not in (1, 4, 8): - raise ValueError( - f"FlashInfer GDN ReplaySSM requires executed width 1, 4, or 8, got " - f"{executed_query_width}" - ) - if len(replayssm_cache) != 3: - raise ValueError( - "FlashInfer GDN ReplaySSM requires exactly three auxiliary rings" - ) - if mixed_qkv.dtype != torch.bfloat16 or checkpoint_state.dtype not in ( - torch.bfloat16, - torch.float16, - ): - raise ValueError( - "FlashInfer GDN ReplaySSM requires bfloat16 inputs and BF16 or FP16 state" - ) - native_stp = executed_query_width == 1 and checkpoint_state.dtype == torch.bfloat16 - # BF16 STP's public contract is T=1, but its serving fast path consumes a - # caller-owned zero-padded T=4 buffer. Packing a shared QKV allocation here - # preserves the common token stride required by block-strided cache pools. - packed_width = 4 if native_stp else executed_query_width - uniform = not native_stp and uniform_query_length == packed_width - # STP metadata permits only zero or one token per row. Eager zero-length - # rows use the packer's fallback. Captured padding can have input storage; - # its negative state index makes FlashInfer skip that row. - single_token = native_stp + uniform = uniform_query_length == executed_width packed_qkv = pack_replayssm_rows( - mixed_qkv, - query_start_loc, - packed_width, - offsets, - uniform=uniform, - single_token=single_token, - # FlashInfer accepts Q/K/V views with a common aligned token stride. - # Gates keep their compact layout independently of this QKV path. - allow_strided_rows=True, + mixed_qkv, query_start_loc, executed_width, offsets, uniform=uniform ) - # CuTe requires every input pointer to be 16-byte aligned. Packing the two - # gates together and splitting them can leave ``b`` at a small byte offset - # into the shared allocation when the local head count is not a multiple of - # eight. Give each gate its own aligned allocation instead. - # The MTP kernel advances every padded lane. Make those lanes exact - # recurrent no-ops: a=-inf gives zero decay after softplus(a + dt_bias), - # while zero q/k/v and b give a zero state update. BF16 STP has no ragged real - # rows, so only invalid graph-padding requests can be padded here; their - # negative state index skips the kernel CTA. STP requires three zero - # input rows after the real token. It returns and commits only that token. + # Padded token lanes must not change the state: a=-inf makes the decay + # exp(-softplus(a + dt_bias) * exp(A_log)) one, and zero q/k/v/b make the + # update zero. a and b are packed separately so each gets an aligned + # allocation. packed_a = pack_replayssm_rows( a, query_start_loc, - packed_width, + executed_width, offsets, - padding_value=0.0 if native_stp else -torch.inf, + padding_value=-torch.inf, uniform=uniform, - single_token=single_token, ) packed_b = pack_replayssm_rows( - b, - query_start_loc, - packed_width, - offsets, - uniform=uniform, - single_token=single_token, + b, query_start_loc, executed_width, offsets, uniform=uniform ) local_k_dim = num_k_heads * head_k_dim local_v_dim = num_v_heads * head_v_dim q_flat, k_flat, v_flat = packed_qkv.split( (local_k_dim, local_k_dim, local_v_dim), dim=-1 ) - q = q_flat.view(packed_qkv.size(0), packed_width, num_k_heads, head_k_dim) + num_rows = packed_qkv.size(0) + q = q_flat.view(num_rows, executed_width, num_k_heads, head_k_dim) k = k_flat.view_as(q) - v = v_flat.view(packed_qkv.size(0), packed_width, num_v_heads, head_v_dim) - # vLLM reserves block zero for padding, while FlashInfer exits a null-row - # CTA only for a negative index. Gather trackers through the safe padding - # slot, then translate every non-live row to FlashInfer's -1 sentinel. + v = v_flat.view(num_rows, executed_width, num_v_heads, head_v_dim) + # vLLM pads with block zero; FlashInfer skips only negative state indices. + # Gather cursors through the padding slot, then pass -1 for padding rows. valid_slots = state_indices > NULL_BLOCK_ID safe_slots = torch.where(valid_slots, state_indices, NULL_BLOCK_ID).long() fi_state_indices = torch.where(valid_slots, state_indices, -1) hist_len = num_committed.index_select(0, safe_slots).masked_fill(~valid_slots, 0) cache_base = ring_start.index_select(0, safe_slots).masked_fill(~valid_slots, 0) u_cache, k_cache, g_cache = replayssm_cache + # Zero, not empty: padded token rows keep flowing through the model. output = torch.zeros( - packed_qkv.size(0), - executed_query_width, + num_rows, + executed_width, num_v_heads, head_v_dim, dtype=mixed_qkv.dtype, device=mixed_qkv.device, ) - - if kernel is None: - kernel = check_gdn_replayssm_dependencies( - executed_query_width, False, checkpoint_state.dtype - ) - if native_stp: - # STP updates these cursor copies. Every layer must start from the - # same pre-step values; model-wide postprocess publishes the update. - mode_kwargs = {"prepadded": True} - flush_min = GDN_REPLAY_LOGICAL_WINDOW - 1 - else: - mode_kwargs = {"restart_hist_on_flush": False} - flush_min = GDN_REPLAY_LOGICAL_WINDOW + 1 - executed_query_width kernel( - # CuTe rejects Parameters that advertise autograd tracking. Fresh - # detached wrappers also prevent stale FlashInfer casts after in-place - # weight reloads. Captured casts read the live parameter storage. + # Detached views, not the Parameters: CuTe rejects tensors that require + # grad, and FlashInfer caches its dtype cast by tensor object. A new + # view per call picks up in-place weight reloads. A_log=A_log.detach(), a=packed_a, dt_bias=dt_bias.detach(), @@ -282,15 +188,9 @@ def run_gdn_replayssm( g_cache=g_cache, hist_len=hist_len, cache_base=cache_base, - flush_min=flush_min, - **mode_kwargs, + # Fold history into the checkpoint once history plus this launch's + # tokens would exceed the logical window. + flush_min=GDN_REPLAYSSM_LOGICAL_WINDOW + 1 - executed_width, + restart_hist_on_flush=False, ) return output.flatten(0, 1).index_select(0, output_indices.long()) - - -__all__ = [ - "GDN_REPLAY_LOGICAL_WINDOW", - "check_gdn_replayssm_dependencies", - "pack_replayssm_rows", - "run_gdn_replayssm", -] diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 10e59e234577..34689e5c12f8 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +import os from collections.abc import Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Literal, TypeAlias @@ -28,28 +29,49 @@ ConvStateLayoutType = Literal["SD", "DS"] -def gdn_replayssm_geometry( - num_speculative_tokens: int, state_dtype: torch.dtype = torch.bfloat16 -) -> tuple[int, int]: - """Return the selected GDN provider's query width and physical ring slots. +GDN_REPLAYSSM_LOGICAL_WINDOW = 16 +"""Committed tokens per request kept in the GDN ReplaySSM rings. Older history +is folded into the checkpoint state.""" +GDN_REPLAYSSM_RING_SLOTS = 32 +"""Physical u/k/G ring slots per request. FlashInfer's UMMA kernel appends up +to eight tokens after the 16-token window and takes a 32-slot ring.""" - Both paths retain at most 16 logical history records. MTP needs extra - physical slots for verification writes while other blocks read history. - BF16 STP folds the current token. FP16 UMMA STP uses the MTP ring contract. + +def gdn_replayssm_executed_width(num_speculative_tokens: int) -> int: + """Return the tokens per request that one GDN ReplaySSM launch computes. + + Shorter rows are padded with no-op tokens up to this width. """ if num_speculative_tokens == 0: - return 1, 32 if state_dtype == torch.float16 else 16 + return 1 if 1 <= num_speculative_tokens <= 3: - return 4, 32 + return 4 if 4 <= num_speculative_tokens <= 7: - return 8, 32 + return 8 raise ValueError( - "FlashInfer GDN ReplaySSM supports 0 through 7 speculative " - "tokens by padding MTP launches to width 4 or 8; got " + "FlashInfer GDN ReplaySSM supports 0 through 7 speculative tokens; got " f"{num_speculative_tokens}" ) +def configure_flashinfer_gdn_env() -> None: + """Select FP16 state in FlashInfer's GDN prefix materializer. + + FlashInfer reads ``GDN_UCACHE_STATE_DTYPE`` once, when ``flashinfer`` is + first imported, so it must be set before vLLM starts its workers. + + Raises: + ValueError: The environment already selects another state dtype. + + """ + state = os.environ.setdefault("GDN_UCACHE_STATE_DTYPE", "fp16") + if state.strip().lower() not in ("fp16", "float16", "half"): + raise ValueError( + f"GDN_UCACHE_STATE_DTYPE={state!r} conflicts with the FP16 recurrent " + "state cache that GDN ReplaySSM uses; unset it" + ) + + def mamba_state_alignment(vllm_config: "VllmConfig") -> int: # FlashInfer loads state and ring caches with 16-byte cp.async instructions. return 16 if vllm_config.use_flashinfer_replayssm else 1 @@ -339,15 +361,15 @@ def gated_delta_net_replayssm_ring_shapes( num_v_heads: int, head_k_dim: int, head_v_dim: int, - ring_slots: int = 32, ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: """Return FlashInfer GDN ``(u, k, G)`` ring shapes.""" local_k_heads = divide(num_k_heads, tp_world_size) local_v_heads = divide(num_v_heads, tp_world_size) + slots = GDN_REPLAYSSM_RING_SLOTS return ( - (local_v_heads, ring_slots, head_v_dim), - (local_k_heads, ring_slots, head_k_dim), - (local_v_heads, ring_slots), + (local_v_heads, slots, head_v_dim), + (local_k_heads, slots, head_k_dim), + (local_v_heads, slots), ) @classmethod diff --git a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py index bebadd4e8ae3..435b2273673c 100644 --- a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py +++ b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py @@ -451,31 +451,6 @@ def selective_state_update_replayssm_flashinfer( ) -@cache -def _load_gdn_replayssm_materialize( - state_dtype: torch.dtype = torch.bfloat16, -) -> Callable[..., None]: - try: - from flashinfer.gdn_kernels.gdn_prefix_materialize import ( - gdn_prefix_materialize, - state_ty_torch, - ) - except ImportError as e: - raise ImportError( - "FlashInfer GDN ReplaySSM prefix caching requires " - "flashinfer.gdn_kernels.gdn_prefix_materialize" - ) from e - if state_ty_torch() != state_dtype: - dtype_name = "fp16" if state_dtype == torch.float16 else "bf16" - raise ValueError( - f"FlashInfer GDN prefix materialization loaded state dtype " - f"{state_ty_torch()}, but vLLM requires {state_dtype}. Set " - f"GDN_UCACHE_STATE_DTYPE={dtype_name} before process startup. " - "Changing the environment after importing FlashInfer is insufficient." - ) - return gdn_prefix_materialize - - def initialize_mamba_ssu_backend( mamba_config: MambaConfig, kv_cache_config: KVCacheConfig, diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index a459fec5e91f..20ed752456b5 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -53,7 +53,6 @@ MambaStateCopyFuncCalculator, MambaStateDtypeCalculator, MambaStateShapeCalculator, - gdn_replayssm_geometry, ) from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -428,16 +427,12 @@ def get_mamba_state_shape_from_config( ) if vllm_config.is_gdn_replayssm_enabled(): - _, ring_slots = gdn_replayssm_geometry( - num_spec, cls.get_mamba_state_dtype_from_config(vllm_config)[1] - ) shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, hf_config.linear_key_head_dim, hf_config.linear_value_head_dim, - ring_slots=ring_slots, ) return shapes diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index 511893dbb791..bbe1d796a614 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import os from collections.abc import Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, Any @@ -9,9 +8,8 @@ import torch from vllm.logger import init_logger -from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_geometry +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_executed_width from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( - _load_gdn_replayssm_materialize, flashinfer_replayssm_autotune_supported, ) @@ -79,6 +77,7 @@ def _temporary_replayssm_autotune_state( *, prepare_dummy_block_tables: bool = True, ) -> Iterator[None]: + """Run FlashInfer ReplaySSM dummy steps, then restore the touched state.""" reset_tensors: list[torch.Tensor] = [] reset_trackers: dict[int, torch.Tensor] = {} for module in runner.get_model().modules(): @@ -253,38 +252,34 @@ def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None: def _warm_gdn_replayssm_materializer( runner: "GPUModelRunner", max_num_reqs: int ) -> None: - """Compile every distinct GDN prefix-materializer specialization.""" - if runner.vllm_config.cache_config.mamba_cache_mode != "align": - return + """Compile each distinct GDN prefix-materializer specialization. + + FlashInfer keys this kernel on the state and ring shapes and strides, so + one call per distinct layout compiles every variant the model uses. + """ + from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention + from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_materializer, + ) + materialize = load_gdn_replayssm_materializer() warmed: set[tuple[Any, ...]] = set() for module in runner.get_model().modules(): - if not getattr(module, "use_flashinfer_replayssm", False) or getattr( - module, "replayssm_executed_query_width", None - ) not in (1, 4, 8): + if not (isinstance(module, GatedDeltaNetAttention) and module.use_replayssm): continue state = module.kv_cache[1] - u_cache, k_cache, g_cache = module.kv_cache[2:5] - signature = ( - state.device, - state.dtype, - tuple(state.shape), - tuple(state.stride()), - tuple(k_cache.shape), - tuple(k_cache.stride()), - tuple(u_cache.shape), - tuple(u_cache.stride()), - tuple(g_cache.shape), - tuple(g_cache.stride()), + u_cache, k_cache, g_cache = module.replayssm_rings + signature = tuple( + (t.device, t.dtype, tuple(t.shape), tuple(t.stride())) + for t in (state, u_cache, k_cache, g_cache) ) if signature in warmed: continue - materialize = _load_gdn_replayssm_materialize(state.dtype) warmed.add(signature) + # Every request index is inactive, so the kernel touches no state. inactive = torch.full( (max_num_reqs,), -1, dtype=torch.int32, device=state.device ) - zeros = torch.zeros_like(inactive) materialize( state=state, src_slots=inactive, @@ -292,103 +287,16 @@ def _warm_gdn_replayssm_materializer( k_cache=k_cache, u_cache=u_cache, g_cache=g_cache, - cache_base=zeros, + cache_base=torch.zeros_like(inactive), count=inactive, active_request_indices=inactive, num_active=torch.zeros(1, dtype=torch.int32, device=state.device), ) -def _warm_gdn_replayssm_mixed_decode( - runner: "GPUModelRunner", executed_query_width: int -) -> None: - """Warm BF16 MTP layouts and launch tactics reachable outside full graphs.""" - if executed_query_width == 1: - return - from flashinfer.gdn_kernels.device_target import gdn_device_target - - from vllm.model_executor.layers.mamba.gdn.replayssm import run_gdn_replayssm - - max_num_reqs = min(runner.scheduler_config.max_num_seqs, runner.max_num_tokens) - warmed: set[tuple[Any, ...]] = set() - for module in runner.get_model().modules(): - if ( - not getattr(module, "use_flashinfer_replayssm", False) - or getattr(module, "replayssm_executed_query_width", None) - != executed_query_width - ): - continue - state, u_cache, k_cache, g_cache = module.kv_cache[1:5] - if state.dtype != torch.bfloat16: - continue - from flashinfer.gdn_kernels import gdn_decode_bf16_wy_ucache_flush as provider - - if not getattr(provider, "SUPPORTS_DYNAMIC_STRIDED_SHAPES", False): - return - signature = tuple( - (t.device, t.dtype, t.shape[1:], t.stride()) - for t in (state, u_cache, k_cache, g_cache) - ) - if signature in warmed: - continue - warmed.add(signature) - _, num_v_heads, head_v_dim, head_k_dim = state.shape - num_k_heads = k_cache.shape[1] - qkv_width = 2 * num_k_heads * head_k_dim + num_v_heads * head_v_dim - strides = [qkv_width] # Mixed requests and ragged rows produce compact QKV. - if runner.vllm_config.num_speculative_tokens + 1 == executed_query_width: - strides.append(qkv_width + num_v_heads * head_v_dim) # QKVZ view. - batches = {1} - if int(os.environ.get("GDN_WY_MBP") or 0) <= 0: - # FlashInfer 0.7.3 HMMA policy: min(ceil(HV * B / SM) + 1, 7). - # No provider selector exposes it. Warm one batch per reachable tactic. - num_sms = gdn_device_target(state.device).num_sms - batches.update(i * num_sms // num_v_heads + 1 for i in range(1, 6)) - for num_reqs in sorted(b for b in batches if b <= max_num_reqs): - num_tokens = num_reqs * executed_query_width - inactive = torch.zeros(num_reqs, dtype=torch.int32, device=state.device) - query_start = torch.arange( - 0, - num_tokens + 1, - executed_query_width, - dtype=torch.int32, - device=state.device, - ) - gates = torch.zeros( - num_tokens, num_v_heads, dtype=torch.bfloat16, device=state.device - ) - for stride in strides: - qkv = torch.zeros( - num_tokens, stride, dtype=torch.bfloat16, device=state.device - )[:, :qkv_width] - # Null slots become -1 in the adapter. The provider exits before - # reading or writing state, including uncommitted cache pages. - run_gdn_replayssm( - mixed_qkv=qkv, - a=gates, - b=gates, - A_log=module.A_log, - dt_bias=module.dt_bias, - checkpoint_state=state, - replayssm_cache=(u_cache, k_cache, g_cache), - ring_start=module._replayssm_ring_start, - num_committed=module._replayssm_prev_num_accepted, - query_start_loc=query_start, - state_indices=inactive, - output_indices=torch.empty( - 0, dtype=torch.long, device=state.device - ), - executed_query_width=executed_query_width, - num_k_heads=num_k_heads, - num_v_heads=num_v_heads, - head_k_dim=head_k_dim, - head_v_dim=head_v_dim, - uniform_query_length=executed_query_width, - ) - - def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: - """Compile GDN replay graph buckets and eager layout/tactic combinations.""" + """Run one uniform decode step per captured graph size, plus the + materializer, so FlashInfer compiles before serving.""" config = runner.vllm_config if not config.is_gdn_replayssm_enabled(): return @@ -398,9 +306,8 @@ def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: ) v2_runner: Any = runner - executed_query_width, _ = gdn_replayssm_geometry(config.num_speculative_tokens) query_len = config.num_speculative_tokens + 1 - assert query_len <= executed_query_width + assert query_len <= gdn_replayssm_executed_width(config.num_speculative_tokens) max_num_reqs = min( runner.scheduler_config.max_num_seqs, runner.max_num_tokens // query_len, @@ -447,5 +354,5 @@ def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: with _temporary_replayssm_autotune_state(runner, max_num_reqs): for num_reqs in sorted(capture_reqs): runner._dummy_run(num_tokens=num_reqs * query_len, **decode_kwargs) - _warm_gdn_replayssm_mixed_decode(runner, executed_query_width) - _warm_gdn_replayssm_materializer(runner, max_num_reqs) + if config.cache_config.mamba_cache_mode == "align": + _warm_gdn_replayssm_materializer(runner, max_num_reqs) diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 8dde6c03db1b..1ac4a6bb7843 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -450,8 +450,8 @@ class CommonAttentionMetadata: is_last_prefill: torch.Tensor | None = None """(batch_size,) bool tensor: True if this scheduled chunk reaches the end - of a request's prompt. Used with ``is_prefilling`` by recurrent backends - that can safely route only the final one-token prompt chunk as decode.""" + of a request's prompt. GDN ReplaySSM replays such a chunk as a decode row + when it is decode-shaped; earlier chunks must update the canonical state.""" seq_lens_cpu_upper_bound: torch.Tensor | None = None """(batch_size,) CPU upper bound on seq_lens. Precise for prefill rows diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 1bfc5a570042..183be3026a77 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -4,8 +4,9 @@ import math from dataclasses import dataclass, replace -from typing import Literal +from typing import Literal, TypeVar +import numpy as np import torch from vllm._aiter_ops import rocm_aiter_ops @@ -14,7 +15,7 @@ MambaPrefillCheckpointBuilder, MambaPrefillCheckpointMetadata, ) -from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_geometry +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_executed_width from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -91,6 +92,32 @@ def supports_device_cpu_query_lens_mismatch(cls) -> bool: return True +ArrayT = TypeVar("ArrayT", np.ndarray, torch.Tensor) + + +def gdn_replayssm_prefill_rows( + *, + is_prefilling: ArrayT, + is_last_prefill: ArrayT, + has_prior_state: ArrayT, + query_lens: ArrayT, + num_draft_tokens: ArrayT | None, +) -> ArrayT: + """Return the rows that GDN ReplaySSM must run as canonical prefill. + + Prompt chunks run as prefill, except the last chunk of a prompt that + already has state when it is shaped like a decode row: one token, or one + token plus its draft tokens. That chunk replays instead. Accepts NumPy + arrays or CPU tensors. + """ + decode_shaped = query_lens == 1 + if num_draft_tokens is not None: + decode_shaped = decode_shaped | ( + (num_draft_tokens >= 0) & (query_lens == num_draft_tokens + 1) + ) + return is_prefilling & ~(is_last_prefill & has_prior_state & decode_shaped) + + @dataclass class GDNAttentionMetadata: num_prefills: int @@ -122,10 +149,10 @@ class GDNAttentionMetadata: num_accepted_tokens: torch.Tensor | None = None # shape: [batch,] uniform_spec_sequence_length: int | None = None # None for ragged batches - # FlashInfer GDN ReplaySSM packs real-token rows into T=1/4/8 launches and - # gathers these dense output positions back afterward. + # GDN ReplaySSM routes every replayed decode row, with or without draft + # tokens, through the spec_* fields above. Its kernel pads each row to the + # launch width; these indices select the real-token outputs. replayssm_output_indices: torch.Tensor | None = None - replayssm_executed_query_width: int | None = None # Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices) chunk_indices: torch.Tensor | None = None @@ -199,8 +226,10 @@ def __init__( self.num_spec = 0 self.use_spec_decode: bool = self.num_spec > 0 self.use_gdn_replayssm = vllm_config.is_gdn_replayssm_enabled() - self.replayssm_executed_query_width = ( - gdn_replayssm_geometry(self.num_spec)[0] if self.use_gdn_replayssm else None + self.replayssm_executed_width = ( + gdn_replayssm_executed_width(self.num_spec) + if self.use_gdn_replayssm + else None ) self._init_reorder_batch_threshold(1, self.use_spec_decode) @@ -383,22 +412,18 @@ def build( # type: ignore[override] "sequence-length metadata" ) query_lens_cpu_all = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] - decode_rows = query_lens_cpu_all == 1 - if num_decode_draft_tokens_cpu is not None: - decode_rows |= (num_decode_draft_tokens_cpu >= 0) & ( - query_lens_cpu_all == num_decode_draft_tokens_cpu + 1 - ) - assert self.replayssm_executed_query_width is not None - replayssm_prefilling = m.is_prefilling & ~( - m.is_last_prefill - & (m.seq_lens_cpu_upper_bound > query_lens_cpu_all) - & decode_rows + assert self.replayssm_executed_width is not None + prefill_rows = gdn_replayssm_prefill_rows( + is_prefilling=m.is_prefilling, + is_last_prefill=m.is_last_prefill, + has_prior_state=m.seq_lens_cpu_upper_bound > query_lens_cpu_all, + query_lens=query_lens_cpu_all, + num_draft_tokens=num_decode_draft_tokens_cpu, ) - replay_width = self.replayssm_executed_query_width spec_sequence_masks_cpu = ( - ~replayssm_prefilling + ~prefill_rows & (query_lens_cpu_all > 0) - & (query_lens_cpu_all <= replay_width) + & (query_lens_cpu_all <= self.replayssm_executed_width) ) num_spec_decodes = int(spec_sequence_masks_cpu.sum().item()) if num_spec_decodes == 0: @@ -448,6 +473,8 @@ def build( # type: ignore[override] no_prior_state &= m.is_prefilling else: no_prior_state = torch.zeros_like(no_prior_state) + # Under GDN ReplaySSM every decode row replays above, so the rows + # here keep the scheduler's prefill flag. num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( m @@ -511,6 +538,7 @@ def build( # type: ignore[override] num_decode_tokens = 0 if num_prefills == 0 and num_decodes == 0: + # Replay rows can be shorter than num_spec + 1 tokens. spec_token_size = ( query_start_loc_cpu[-1].item() if self.use_gdn_replayssm @@ -593,14 +621,14 @@ def build( # type: ignore[override] else: num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] if self.use_gdn_replayssm: - assert self.replayssm_executed_query_width is not None + assert self.replayssm_executed_width is not None replay_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu] row_offsets = torch.arange( - self.replayssm_executed_query_width, dtype=torch.int32 + self.replayssm_executed_width, dtype=torch.int32 ).unsqueeze(0) dense_indices = ( torch.arange(num_spec_decodes, dtype=torch.int32).unsqueeze(1) - * self.replayssm_executed_query_width + * self.replayssm_executed_width + row_offsets ) replayssm_output_indices = async_tensor_h2d( @@ -761,7 +789,8 @@ def build( # type: ignore[override] self.replayssm_output_indices[:output_count].copy_( replayssm_output_indices, non_blocking=True ) - # Captured graphs retain a longer view when live rows shrink. + # A captured graph keeps its capture-time view length; zero the + # stale tail so its extra indices stay in bounds. self.replayssm_output_indices[output_count:].zero_() replayssm_output_indices = self.replayssm_output_indices[:output_count] @@ -809,7 +838,6 @@ def build( # type: ignore[override] num_accepted_tokens=num_accepted_tokens, uniform_spec_sequence_length=uniform_spec_sequence_length, replayssm_output_indices=replayssm_output_indices, - replayssm_executed_query_width=self.replayssm_executed_query_width, nums_dict=nums_dict, batch_ptr=batch_ptr, token_chunk_offset_ptr=token_chunk_offset_ptr, diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index 2945bb9ac0b8..58b2fc842ed1 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -397,7 +397,9 @@ def prepare_attn( assert self._mamba_ctx is not None replay = self._mamba_ctx.replayssm assert replay is not None - # Speculative CPU positions are upper bounds, so keep MTP conservative. + # Skip GDN's per-layer materialization when no request can reach + # a block boundary. With draft tokens the CPU positions here are + # only upper bounds, so the gate applies to standard decoding. self._replayssm_may_materialize = may_cross_mamba_block_boundary( input_batch.num_computed_tokens_np, input_batch.num_scheduled_tokens, @@ -423,6 +425,7 @@ def prepare_attn( isinstance(mamba_metadata, GDNAttentionMetadata) and mamba_metadata.spec_sequence_masks_cpu is not None ): + # GDN replay rows are the spec rows; all others prefilled. replayssm_prefilling = ~mamba_metadata.spec_sequence_masks_cpu[ :num_reqs ] diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 460529cea462..8b13a19f84e0 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -131,6 +131,7 @@ from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionMetadata, GDNAttentionMetadataBuilder, + gdn_replayssm_prefill_rows, ) from vllm.v1.attention.backends.linear_attn import ( BailingLinearAttentionMetadataBuilder, @@ -3726,12 +3727,14 @@ def _gdn_has_prefill( query: np.ndarray, draft: np.ndarray | None, ) -> bool: - # Match GDNAttentionMetadataBuilder's canonical-prefill classification. - decode_rows = query == 1 - if draft is not None: - decode_rows |= (draft >= 0) & (query == draft + 1) - final_decode = (computed > 0) & (computed + query >= prompt) & decode_rows - return bool(((computed < prompt) & ~final_decode).any()) + prefill_rows = gdn_replayssm_prefill_rows( + is_prefilling=computed < prompt, + is_last_prefill=computed + query >= prompt, + has_prior_state=computed > 0, + query_lens=query, + num_draft_tokens=draft, + ) + return bool(prefill_rows.any()) @staticmethod def _is_uniform_decode( @@ -4242,14 +4245,19 @@ def execute_model( ] # Cache groups share the request split, but own distinct slots. mamba_metadata = attn_metadata[mamba_group.layer_names[0]] - if isinstance(mamba_metadata, GDNAttentionMetadata): - replay_rows = mamba_metadata.spec_sequence_masks_cpu - if replay_rows is not None: - gdn_prefilling = ~replay_rows[:num_reqs] - else: - num_mamba_decodes = mamba_metadata.num_decodes + replay_rows = ( + mamba_metadata.spec_sequence_masks_cpu + if isinstance(mamba_metadata, GDNAttentionMetadata) + else None + ) + if replay_rows is not None: + # GDN replay rows are the spec rows; all others prefilled. + gdn_prefilling = ~replay_rows[:num_reqs] else: - assert isinstance(mamba_metadata, Mamba2AttentionMetadata) + assert isinstance( + mamba_metadata, + (GDNAttentionMetadata, Mamba2AttentionMetadata), + ) num_mamba_decodes = mamba_metadata.num_decodes mamba_utils.stage_postprocess_inputs_to_gpu( mamba_ctx, diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index e05794b41004..93f74f716297 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -1760,6 +1760,9 @@ def postprocess_mamba_gpu( is_prefilling=ctx.is_prefilling_buf.gpu, live_cols=live_cols, num_reqs=num_reqs, + # GDN materializes one layer per launch, so skip steps where no + # request can reach a block boundary. The kernel places boundaries + # from these same pre-step counts. may_materialize=( not ctx.replayssm.materialize_prefixes or ctx.replayssm.mamba_type != MambaAttentionBackendEnum.GDN_ATTN @@ -1792,8 +1795,11 @@ def stage_postprocess_inputs_to_gpu( """Stage the per-request decisions consumed after token acceptance. Generic Mamba uses scheduled, computed, and draft counts. ReplaySSM also - uses the final Mamba attention split to reset trackers after prefill. + uses the final Mamba attention split to reset trackers after prefill: + either ``num_decodes`` leading decode rows or an explicit ``is_prefilling`` + mask (GDN, whose replay rows need not lead the batch). """ + assert num_decodes is None or is_prefilling is None assert ctx.num_scheduled_tokens_buf is not None assert ctx.num_computed_tokens_buf is not None assert ctx.num_draft_tokens_buf is not None diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index 538fb748858c..f340f8e63943 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -11,9 +11,10 @@ import torch from vllm.model_executor.layers.mamba.abstract import MambaBase -from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( - _load_gdn_replayssm_materialize, +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_materializer, ) +from vllm.model_executor.layers.mamba.mamba_utils import GDN_REPLAYSSM_RING_SLOTS from vllm.model_executor.models.utils import extract_layer_index from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -24,7 +25,11 @@ def may_cross_mamba_block_boundary( computed: np.ndarray, scheduled: np.ndarray, block_size: int ) -> bool: - """Bound accepted progress using exact pre-step counts and scheduled lengths.""" + """Whether any request could cross a Mamba block boundary this step. + + A request accepts at most its scheduled tokens, so ``False`` means no prefix + snapshot can be due and postprocess may skip materialization. + """ return bool(((computed + scheduled) // block_size > computed // block_size).any()) @@ -187,7 +192,6 @@ def _postprocess_replayssm_kernel( MATERIALIZE_PREFIXES: tl.constexpr, LIVE_COL_IS_ZERO: tl.constexpr, EXECUTED_QUERY_WIDTH: tl.constexpr, - GDN_STP: tl.constexpr, ) -> None: """Commit a completed step and prepare an optional prefix snapshot.""" batch_idx = tl.program_id(0) @@ -301,42 +305,28 @@ def _postprocess_replayssm_kernel( elif valid_live & valid_query: old_start = tl.load(tracker_start + live_slot) old_committed = tl.load(tracker_committed + live_slot) - if GDN_STP: - # FlashInfer's GDN STP fold absorbs the current token into the - # checkpoint at P == L - 1 and uses a flat 16-slot ring. Layer - # forwards mutate gathered cursor copies; publish the equivalent - # group-owned transition once here. - checkpointed = old_committed >= LOGICAL_WINDOW - 1 - next_start = 0 - next_committed = tl.where(checkpointed, 0, old_committed + 1) - else: - executed_query_len = query_len - if EXECUTED_QUERY_WIDTH > 0: - executed_query_len = EXECUTED_QUERY_WIDTH - checkpointed = old_committed + executed_query_len > LOGICAL_WINDOW - next_start = tl.where( - checkpointed, - (old_start + old_committed) % RING_BUFFER_LEN, - old_start, - ) - next_committed = tl.where(checkpointed, accepted, old_committed + accepted) + # GDN launches pad every row to a fixed width, and its kernel decides + # whether to fold from that width rather than from this row's length. + executed_query_len = query_len + if EXECUTED_QUERY_WIDTH > 0: + executed_query_len = EXECUTED_QUERY_WIDTH + checkpointed = old_committed + executed_query_len > LOGICAL_WINDOW + next_start = tl.where( + checkpointed, + (old_start + old_committed) % RING_BUFFER_LEN, + old_start, + ) + next_committed = tl.where(checkpointed, accepted, old_committed + accepted) tl.store(tracker_start + live_slot, next_start) tl.store(tracker_committed + live_slot, next_committed) if wants_materialize: tl.store(plan_ring_start + batch_idx, next_start, mask=writes_plan) - if GDN_STP: - # On a fold, state already contains the current token and an - # exact count=0 copy publishes the boundary. Otherwise replay - # the old history plus the one accepted STP token. - materialize_count = tl.where( - checkpointed, 0, old_committed + accept_token_bias + 1 - ) - else: - materialize_count = ( - accept_token_bias + 1 + tl.where(checkpointed, 0, old_committed) - ) - tl.store(plan_flush_count + batch_idx, materialize_count, mask=writes_plan) + tl.store( + plan_flush_count + batch_idx, + accept_token_bias + 1 + tl.where(checkpointed, 0, old_committed), + mask=writes_plan, + ) # The published destination is canonical and therefore has no live replay. # When dst_slot aliases live_slot, these stores intentionally supersede the @@ -415,7 +405,7 @@ class ReplaySSMModelContext: logical_window: int ring_buffer_len: int materialize_prefixes: bool - executed_query_width: int + executed_query_width: int # GDN launch width; 0 uses each row's length. mamba_type: MambaAttentionBackendEnum @classmethod @@ -445,44 +435,15 @@ def create( ), "FlashInfer ReplaySSM requires equal materialization geometry" device = first.kv_cache[1].device - logical_window = int(first.replayssm_buffer_len) - executed_query_width = int( - getattr(first, "replayssm_executed_query_width", 0) or 0 - ) - ring_buffer_len = first.kv_cache[2].size(2) + executed_query_width = 0 if mamba_type == MambaAttentionBackendEnum.GDN_ATTN: - if logical_window != 16 or executed_query_width not in (1, 4, 8): - raise ValueError( - "FlashInfer GDN ReplaySSM requires logical window 16 and " - "executed query width 1, 4, or 8; got " - f"window={logical_window}, width={executed_query_width}" - ) - expected_ring_buffer_len = ( - 16 - if executed_query_width == 1 - and first.kv_cache[1].dtype == torch.bfloat16 - else 32 - ) - if ring_buffer_len != expected_ring_buffer_len: + # Every GDN layer derives its launch width from the same config. + executed_query_width = first.replayssm_executed_width + if first.kv_cache[2].size(2) != GDN_REPLAYSSM_RING_SLOTS: raise ValueError( - "FlashInfer GDN ReplaySSM ring depth does not match its " - f"executed width: width={executed_query_width} requires " - f"{expected_ring_buffer_len} slots, got {ring_buffer_len}" + f"FlashInfer GDN ReplaySSM requires {GDN_REPLAYSSM_RING_SLOTS} " + f"ring slots, got {first.kv_cache[2].size(2)}" ) - for mixer in mixers[1:]: - mixer_width = int( - getattr(mixer, "replayssm_executed_query_width", 0) or 0 - ) - mixer_ring_len = mixer.kv_cache[2].size(2) - if ( - int(mixer.replayssm_buffer_len) != logical_window - or mixer_width != executed_query_width - or mixer_ring_len != ring_buffer_len - ): - raise ValueError( - "all GDN ReplaySSM layers in a cache group must share " - "the same logical window, executed width, and ring depth" - ) block_tables = [table for _, _, table in grouped] src_slots = torch.full( (len(mixers), max_num_reqs), @@ -563,11 +524,13 @@ def create( ) def _materialize(self) -> None: - """Publish canonical prefix snapshots with the selected backend.""" + """Publish canonical prefix snapshots. + + Mamba2 publishes every layer in one FlashInfer call. FlashInfer's GDN + materializer takes one layer per call. + """ if self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: - self.materialize_gdn( - _load_gdn_replayssm_materialize(self.mixers[0].kv_cache[1].dtype) - ) + self._materialize_gdn() return assert self.materialize_tables is not None first = self.mixers[0] @@ -600,8 +563,8 @@ def _materialize(self) -> None: philox_rounds=philox_rounds, ) - def materialize_gdn(self, materialize_fn: Callable[..., None]) -> None: - """Publish one GDN recurrent-state snapshot per layer.""" + def _materialize_gdn(self) -> None: + materialize_fn = load_gdn_replayssm_materializer() for layer_idx, mixer in enumerate(self.mixers): u_cache, k_cache, g_cache = mixer.kv_cache[2:5] materialize_fn( @@ -684,11 +647,6 @@ def postprocess( MATERIALIZE_PREFIXES=self.materialize_prefixes, LIVE_COL_IS_ZERO=live_cols is None, EXECUTED_QUERY_WIDTH=self.executed_query_width, - GDN_STP=( - self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN - and self.executed_query_width == 1 - and self.ring_buffer_len == 16 - ), ) if not self.materialize_prefixes or not may_materialize: return From 08ed00730b7dfde5374cda7cbb13014c63238337 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Fri, 6 Nov 2026 17:41:07 +0100 Subject: [PATCH 09/14] [Mamba] Fix FP16 GDN ReplaySSM review regressions Propagate the FP16 materializer setting to Ray engine actors and preserve aligned strided QKV views. Select native recurrence tests on B200 and use real configuration constructors in the FP16 validation tests. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- .buildkite/test_areas/kernels.yaml | 4 ++ tests/kernels/mamba/test_gdn_replayssm.py | 49 ++++++++++++++ tests/test_config.py | 67 +++++++++++++------ tests/test_ray_env.py | 10 +++ .../layers/mamba/gdn/replayssm.py | 21 +++++- vllm/ray/ray_env.py | 1 + 6 files changed, 128 insertions(+), 24 deletions(-) diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 7a4ef3fc8fb1..7edcc079163e 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -742,6 +742,8 @@ steps: - vllm/platforms/cuda.py - vllm/cute_utils/ - vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/ + - vllm/model_executor/layers/mamba/gdn/ + - vllm/model_executor/layers/mamba/mamba_utils.py - vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_sm100.py - cmake/external_projects/fmha_sm100.cmake - vllm/models/minimax_m3/common/sparse_attention.py @@ -758,6 +760,7 @@ steps: - tests/kernels/quantization/nvfp4_utils.py - tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py - tests/kernels/mamba/test_gdn_prefill_cutedsl.py + - tests/kernels/mamba/test_gdn_replayssm.py - tests/kernels/test_bf16x3_router_gemm_cutedsl.py - tests/kernels/test_top_k_per_row.py - csrc/fused_mqa_topk/ @@ -797,6 +800,7 @@ steps: - pytest -v -s tests/kernels/moe/test_flashinfer.py - pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py - pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py + - pytest -v -s tests/kernels/mamba/test_gdn_replayssm.py mirror: amd: automatic_shard: true diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 387c60557dcb..e7bee8735280 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -85,6 +85,55 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): assert padded[1].eq(0).all() +@pytest.mark.parametrize("width", [4, 8]) +def test_uniform_replayssm_passes_qkv_views_and_aligned_gates(width): + tokens, dim = 2 * width, 128 + projection = torch.arange(tokens * 4 * dim).to(torch.bfloat16).view(tokens, -1) + mixed_qkv = projection[:, : 3 * dim] + gates = torch.ones(tokens, 2, dtype=torch.bfloat16) + a, b = gates[:, :1], gates[:, 1:] + + def kernel(**kwargs): + # UMMA makes its own contiguous Q/K/V copies. Keep the packed view. + assert kwargs["q"].data_ptr() == mixed_qkv.data_ptr() + for name, expected in zip(("q", "k", "v"), mixed_qkv.split(dim, -1)): + torch.testing.assert_close( + kwargs[name].flatten(0, 2), expected, rtol=0, atol=0 + ) + for name, expected in (("a", a), ("b", b)): + assert kwargs[name].is_contiguous() + assert kwargs[name].data_ptr() % 16 == 0 + torch.testing.assert_close(kwargs[name].flatten(0, 1), expected) + kwargs["output"].copy_(kwargs["q"]) + + output = run_gdn_replayssm( + kernel=kernel, + executed_width=width, + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=torch.zeros(1), + dt_bias=torch.zeros(1), + checkpoint_state=torch.zeros(3, 1, dim, dim, dtype=torch.float16), + replayssm_cache=( + torch.zeros(3, 1, 32, dim, dtype=torch.bfloat16), + torch.zeros(3, 1, 32, dim, dtype=torch.bfloat16), + torch.zeros(3, 1, 32), + ), + ring_start=torch.zeros(3, dtype=torch.int32), + num_committed=torch.zeros(3, dtype=torch.int32), + query_start_loc=torch.tensor([0, width, tokens], dtype=torch.int32), + state_indices=torch.tensor([1, 2], dtype=torch.int32), + output_indices=torch.arange(tokens), + num_k_heads=1, + num_v_heads=1, + head_k_dim=dim, + head_v_dim=dim, + uniform_query_length=width, + ) + torch.testing.assert_close(output[:, 0], mixed_qkv[:, :dim], rtol=0, atol=0) + + def _reference_gdn_step( state, q, diff --git a/tests/test_config.py b/tests/test_config.py index c56284017369..3ea4b8e4f267 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -44,6 +44,7 @@ WatermarkConfig, update_config, ) +from vllm.config.cache import MambaCacheMode, MambaDType from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig from vllm.config.kernel import IrOpPriorityConfig from vllm.config.load import LoadConfig @@ -158,49 +159,71 @@ def test_flashinfer_replayssm_validates_config( VllmConfig.validate_mamba_cached_kernel(config) -def _gdn_replayssm_config(mode: str, ssm_dtype: str = "float16") -> SimpleNamespace: - return SimpleNamespace( - cache_config=SimpleNamespace( +def _gdn_replayssm_config( + model_path: Path, + mode: MambaCacheMode, + ssm_dtype: MambaDType = "float16", +) -> VllmConfig: + from transformers import Qwen3_5TextConfig + + Qwen3_5TextConfig( + architectures=["Qwen3_5ForCausalLM"], + hidden_size=256, + intermediate_size=512, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=128, + linear_num_key_heads=1, + linear_num_value_heads=2, + linear_key_head_dim=128, + linear_value_head_dim=128, + layer_types=["linear_attention", "full_attention"], + vocab_size=128, + max_position_embeddings=128, + ).save_pretrained(model_path) + return VllmConfig( + model_config=ModelConfig( + model=str(model_path), + tokenizer_mode="skip", + dtype="bfloat16", + max_model_len=128, + enforce_eager=True, + ), + cache_config=CacheConfig( use_replayssm=True, - use_kda_recoverssm=False, replayssm_buffer_len=16, mamba_cache_mode=mode, enable_prefix_caching=mode == "align", mamba_cache_dtype="auto", mamba_ssm_cache_dtype=ssm_dtype, ), - num_speculative_tokens=3, - speculative_config=None, - model_config=SimpleNamespace( - architecture="Qwen3_5MoeForCausalLM", - dtype=torch.bfloat16, - hf_text_config=SimpleNamespace( - linear_key_head_dim=128, linear_value_head_dim=128 - ), - ), - mamba_config=SimpleNamespace(enable_stochastic_rounding=False), - parallel_config=SimpleNamespace(pipeline_parallel_size=1, use_ubatching=False), - kv_transfer_config=None, + scheduler_config=SchedulerConfig(max_model_len=128, is_encoder_decoder=False), + parallel_config=ParallelConfig(distributed_executor_backend="uni"), + device_config=DeviceConfig(device="cpu"), ) -def test_gdn_replayssm_requires_fp16_state(): +def test_gdn_replayssm_requires_fp16_state(tmp_path): with pytest.raises(ValueError, match="--mamba-ssm-cache-dtype float16"): - VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("none", "bfloat16")) + _gdn_replayssm_config(tmp_path, "none", "bfloat16") -def test_gdn_replayssm_align_configures_flashinfer_materializer(monkeypatch): +def test_gdn_replayssm_align_configures_flashinfer_materializer(tmp_path, monkeypatch): """--use-replayssm selects FlashInfer's FP16 materializer before import.""" # Register the variable so monkeypatch removes what the validator sets. monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "") monkeypatch.delenv("GDN_UCACHE_STATE_DTYPE") - VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("none")) + config = _gdn_replayssm_config(tmp_path, "none") + assert config.is_gdn_replayssm_enabled() + assert config.use_flashinfer_replayssm assert "GDN_UCACHE_STATE_DTYPE" not in os.environ - VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("align")) + config = _gdn_replayssm_config(tmp_path, "align") + assert config.is_gdn_replayssm_enabled() assert os.environ["GDN_UCACHE_STATE_DTYPE"] == "fp16" monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "bf16") with pytest.raises(ValueError, match="GDN_UCACHE_STATE_DTYPE='bf16'"): - VllmConfig.validate_gdn_replayssm(_gdn_replayssm_config("align")) + _gdn_replayssm_config(tmp_path, "align") def _write_json(path: Path, value: object) -> None: diff --git a/tests/test_ray_env.py b/tests/test_ray_env.py index 945b2d80b691..5e2d433201e0 100644 --- a/tests/test_ray_env.py +++ b/tests/test_ray_env.py @@ -57,6 +57,16 @@ def test_pythonhashseed_in_result(self): result = get_env_vars_to_copy() assert "PYTHONHASHSEED" in result + def test_gdn_materializer_state_reaches_engine_actor(self, monkeypatch): + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "fp16") + monkeypatch.delenv("VLLM_RAY_EXTRA_ENV_VARS_TO_COPY", raising=False) + monkeypatch.delenv("VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY", raising=False) + names = get_env_vars_to_copy( + destination="EngineCoreActor", exclude_vars=WORKER_SPECIFIC_ENV_VARS + ) + actor_env = {name: os.environ[name] for name in names if name in os.environ} + assert actor_env.get("GDN_UCACHE_STATE_DTYPE") == "fp16" + # --------------------------------------------------------------------------- # User-supplied extensions diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index a9902d3e9aaf..fddbd36a95d8 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -58,6 +58,7 @@ def pack_replayssm_rows( offsets: torch.Tensor | None = None, padding_value: float = 0.0, uniform: bool = False, + allow_strided_rows: bool = False, ) -> torch.Tensor: """Pack variable-length token rows into ``[num_rows, executed_width, ...]``. @@ -65,10 +66,20 @@ def pack_replayssm_rows( followed by ``padding_value``. With ``uniform``, every row has exactly ``executed_width`` tokens, so the result is a view of ``values`` when it is contiguous and aligned for CuTe, and a contiguous copy otherwise. + ``allow_strided_rows`` also preserves aligned, non-overlapping 2D rows. """ num_rows = query_start_loc.numel() - 1 if uniform and values.shape[0] == num_rows * executed_width: - if values.data_ptr() % _CUTE_ALIGNMENT_BYTES or not values.is_contiguous(): + aligned_rows = ( + allow_strided_rows + and values.ndim == 2 + and values.stride(1) == 1 + and values.stride(0) >= values.shape[1] + and values.stride(0) * values.element_size() % _CUTE_ALIGNMENT_BYTES == 0 + ) + if values.data_ptr() % _CUTE_ALIGNMENT_BYTES or not ( + values.is_contiguous() or aligned_rows + ): values = values.clone(memory_format=torch.contiguous_format) return values.view(num_rows, executed_width, *values.shape[1:]) if num_rows == 0: @@ -124,7 +135,13 @@ def run_gdn_replayssm( """ uniform = uniform_query_length == executed_width packed_qkv = pack_replayssm_rows( - mixed_qkv, query_start_loc, executed_width, offsets, uniform=uniform + mixed_qkv, + query_start_loc, + executed_width, + offsets, + uniform=uniform, + # UMMA makes contiguous Q/K/V copies after splitting the packed view. + allow_strided_rows=True, ) # Padded token lanes must not change the state: a=-inf makes the decay # exp(-softplus(a + dt_bias) * exp(A_log)) one, and zero q/k/v/b make the diff --git a/vllm/ray/ray_env.py b/vllm/ray/ray_env.py index 2e2ed48b85c7..7f0c3231d91e 100644 --- a/vllm/ray/ray_env.py +++ b/vllm/ray/ray_env.py @@ -44,6 +44,7 @@ } DEFAULT_EXTRA_ENV_VARS: set[str] = { + "GDN_UCACHE_STATE_DTYPE", "PYTHONHASHSEED", } From ae5fbcc3fd446cffee081868da674b7579cd09a5 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Mon, 9 Nov 2026 13:42:19 +0100 Subject: [PATCH 10/14] [Mamba] Restore GDN ReplaySSM startup and simplify tests Accept GDN state-copy layouts and resolve wrapped Mamba cache specs during ReplaySSM initialization. Extend the existing CPU regression through both startup steps. Reduce repeated fixtures and equivalent test cases. Remove obsolete array typing and clarify the FlashInfer requirement. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- tests/kernels/mamba/test_gdn_replayssm.py | 36 ++++-------- tests/kernels/mamba/test_ssu_dispatch.py | 44 ++++++--------- tests/test_config.py | 8 --- .../v1/attention/test_gdn_metadata_builder.py | 17 ++---- tests/v1/worker/test_gpu_batch_ordering.py | 10 +--- tests/v1/worker/test_mamba_utils.py | 55 +++++++++++++++++-- vllm/config/cache.py | 9 +-- vllm/v1/attention/backends/gdn_attn.py | 21 +++---- vllm/v1/worker/mamba_utils.py | 22 +++++--- 9 files changed, 109 insertions(+), 113 deletions(-) diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py index 4bd0bfe336c5..e159f0a06644 100644 --- a/tests/kernels/mamba/test_gdn_replayssm.py +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import builtins import math +import sys from types import ModuleType import pytest @@ -24,14 +24,9 @@ def test_materializer_requires_fp16_state(monkeypatch, loaded_dtype): materialize = lambda **kwargs: None monkeypatch.setattr(provider, "gdn_prefix_materialize", materialize, raising=False) monkeypatch.setattr(provider, "state_ty_torch", lambda: loaded_dtype, raising=False) - original_import = builtins.__import__ - - def import_provider(name, *args, **kwargs): - if name == "flashinfer.gdn_kernels.gdn_prefix_materialize": - return provider - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", import_provider) + monkeypatch.setitem( + sys.modules, "flashinfer.gdn_kernels.gdn_prefix_materialize", provider + ) load_gdn_replayssm_materializer.cache_clear() try: if loaded_dtype == torch.float16: @@ -59,8 +54,8 @@ def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): ) -@pytest.mark.parametrize("width", [4, 8]) -def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): +def test_uniform_replayssm_packing_preserves_alignment_and_padding(): + width = 4 values = torch.arange(2 * width * 4, dtype=torch.bfloat16).view(2 * width, 4) starts = torch.tensor([0, width, 2 * width]) packed = pack_replayssm_rows(values, starts, width, uniform=True) @@ -85,8 +80,8 @@ def test_uniform_replayssm_packing_preserves_alignment_and_padding(width): assert padded[1].eq(0).all() -@pytest.mark.parametrize("width", [4, 8]) -def test_uniform_replayssm_passes_qkv_views_and_aligned_gates(width): +def test_uniform_replayssm_passes_qkv_views_and_aligned_gates(): + width = 4 tokens, dim = 2 * width, 128 projection = torch.arange(tokens * 4 * dim).to(torch.bfloat16).view(tokens, -1) mixed_qkv = projection[:, : 3 * dim] @@ -200,7 +195,7 @@ def _reconstruct_replayssm_state( ) @pytest.mark.parametrize( "width,query_len,accepted", - [(1, 1, 1), (4, 3, 2), (4, 4, 2), (8, 6, 4), (8, 8, 4)], + [(1, 1, 1), (4, 3, 2), (8, 6, 4), (8, 8, 4)], ) @torch.inference_mode() def test_gdn_replayssm_output_and_accepted_state(width, query_len, accepted): @@ -259,16 +254,9 @@ def random_tensor(*shape): a, b = gates[:, 0], gates[:, 1] if capture: # Captured padding has input storage but no token or live state. - padding_tokens = width - mixed_qkv = torch.cat( - ( - mixed_qkv, - torch.ones_like(mixed_qkv[:1]).expand(padding_tokens, -1), - ) - ) - a, b = ( - torch.cat((a, torch.ones_like(a[:1]).expand(padding_tokens, -1))), - torch.cat((b, torch.ones_like(b[:1]).expand(padding_tokens, -1))), + mixed_qkv, a, b = ( + torch.cat((values, torch.ones_like(values[:1]).expand(width, -1))) + for values in (mixed_qkv, a, b) ) inputs = dict( mixed_qkv=mixed_qkv, diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index a902cecd21e4..d0f42e7948ec 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -4,7 +4,6 @@ import os from dataclasses import replace from types import SimpleNamespace -from typing import NamedTuple from unittest.mock import Mock import pytest @@ -315,45 +314,35 @@ def test_replayssm_materializes_only_accepted_boundary_with_compacted_rows( torch.testing.assert_close(states[1], before_shrink[name], atol=0, rtol=0) -class _PostprocessCase(NamedTuple): - ring_slots: int - executed_width: int # 0: fold decisions use the row's query length. - query_len: int - committed: int # Live slot history before the step. - flush_count: int - ring_start: int - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -@pytest.mark.parametrize("post_step", [False, True], ids=["v1", "v2"]) @pytest.mark.parametrize( - "case", + "post_step,ring_slots,executed_width,query_len,committed,flush_count,ring_start", [ # Mamba2: 2 + 4 fits the window, so replay 2 old + 4 accepted tokens. - pytest.param(_PostprocessCase(20, 0, 4, 2, 6, 3), id="mamba2"), + pytest.param(False, 20, 0, 4, 2, 6, 3, id="mamba2-pre-step"), + pytest.param(True, 20, 0, 4, 2, 6, 3, id="mamba2-post-step"), # GDN standard decoding: 16 + 1 > 16 folds; publish only the new token. - pytest.param(_PostprocessCase(32, 1, 1, 16, 1, 19), id="gdn-stp-fold"), + pytest.param(True, 32, 1, 1, 16, 1, 19, id="gdn-stp-fold"), # A 4-token row in a width-8 GDN launch folds at 10 + 8 > 16, although # 10 + 4 would fit. - pytest.param(_PostprocessCase(32, 8, 4, 10, 4, 13), id="gdn-mtp-fold"), - pytest.param(_PostprocessCase(32, 8, 4, 8, 12, 3), id="gdn-mtp-append"), + pytest.param(True, 32, 8, 4, 10, 4, 13, id="gdn-mtp-fold"), + pytest.param(True, 32, 8, 4, 8, 12, 3, id="gdn-mtp-append"), ], ) def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( - post_step, case + post_step, ring_slots, executed_width, query_len, committed, flush_count, ring_start ): - """Skip prefill self-copies, reset trackers, and preserve in-place MTP replay.""" - slots = case.ring_slots + """Skip prefill self-copies, reset trackers, and preserve in-place replay.""" mixer = _replayssm_mixer( kv_cache=( torch.empty(0, device="cuda"), torch.empty(6, 1, 1, 1, device="cuda"), - torch.empty(6, 1, slots, 1, device="cuda"), - torch.empty(6, 1, slots, device="cuda"), - torch.empty(6, 1, slots, 1, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), + torch.empty(6, 1, ring_slots, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), ), ring_start=_cuda_ints([3] * 6), - committed=_cuda_ints([2, 2, 2, 2, case.committed, 2]), + committed=_cuda_ints([2, 2, 2, 2, committed, 2]), A=torch.empty(1, device="cuda"), ) spec = MambaSpec( @@ -374,7 +363,6 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( # Reordered rows: canonical prefill in slot 2; in-place replay in slot 4 # that accepts every token and reaches the block boundary at 32; padded # slot; padded request. - query_len = case.query_len _postprocess_replayssm_kernel[(4, 1)]( _cuda_ints([1, 2, 0, -1]), _cuda_ints([0, 48, 48 + query_len, 64 + query_len, 64 + query_len]) @@ -396,14 +384,14 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( MAX_NUM_REQS=4, MAMBA_BLOCK_SIZE=16, LOGICAL_WINDOW=16, - RING_BUFFER_LEN=slots, + RING_BUFFER_LEN=ring_slots, PAD_SLOT_ID=NULL_BLOCK_ID, QUERY_METADATA_IS_CUMULATIVE=post_step, NUM_COMPUTED_IS_POST_STEP=post_step, HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, - EXECUTED_QUERY_WIDTH=case.executed_width, + EXECUTED_QUERY_WIDTH=executed_width, ) _compact_replayssm_requests_kernel[(1,)]( context.plan_flush_count, @@ -414,8 +402,8 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( BLOCK_SIZE=4, ) - assert context.plan_flush_count.tolist() == [-1, case.flush_count, -1, -1] - assert context.plan_ring_start[1].item() == case.ring_start + assert context.plan_flush_count.tolist() == [-1, flush_count, -1, -1] + assert context.plan_ring_start[1].item() == ring_start assert context.active_request_indices.tolist() == [1, -1, -1, -1] assert context.src_slots[0, 1].item() == 4 assert context.dst_slots[0, 1].item() == 4 diff --git a/tests/test_config.py b/tests/test_config.py index 290dff773cf7..c17490715e52 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -167,18 +167,10 @@ def _gdn_replayssm_config( Qwen3_5TextConfig( architectures=["Qwen3_5ForCausalLM"], - hidden_size=256, - intermediate_size=512, num_hidden_layers=2, - num_attention_heads=2, - num_key_value_heads=1, - head_dim=128, - linear_num_key_heads=1, - linear_num_value_heads=2, linear_key_head_dim=128, linear_value_head_dim=128, layer_types=["linear_attention", "full_attention"], - vocab_size=128, max_position_embeddings=128, ).save_pretrained(model_path) return VllmConfig( diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index fd9842073af6..1cbb1fc1b75d 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -152,12 +152,14 @@ def _create_gdn_builder( num_speculative_tokens: int = 0, full_cuda_graph: bool = False, adaptive_verification: bool = False, + use_replayssm: bool = False, ) -> GDNAttentionMetadataBuilder: """Create a GDNAttentionMetadataBuilder with minimal config.""" vllm_config = create_vllm_config( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE, ) + vllm_config.cache_config.use_replayssm = use_replayssm vllm_config.compilation_config.cudagraph_mode = ( CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE ) @@ -287,19 +289,8 @@ def test_has_initial_state_after_reclassification(): def test_replayssm_captured_indices_remain_valid_after_ragged_batch(): """A smaller FULL graph must not read stale indices from a ragged step.""" - config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE) - config.cache_config.use_replayssm = True - config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE - config.speculative_config = SpeculativeConfig( - method="ngram", num_speculative_tokens=3 - ) - builder = GDNAttentionMetadataBuilder( - kv_cache_spec=MambaSpec( - block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.bfloat16,) - ), - layer_names=["layer.0"], - vllm_config=config, - device=DEVICE, + builder = _create_gdn_builder( + num_speculative_tokens=3, full_cuda_graph=True, use_replayssm=True ) captured = _build(builder, BatchSpec(seq_lens=[80] * 32, query_lens=[4] * 32)) indices = captured.replayssm_output_indices diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index ceb22ded1b4d..d6bc92303b69 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -336,13 +336,7 @@ def test_no_speculator_dispatches_on_query_length_alone(): def test_replayssm_materialization_bound_covers_partial_acceptance(): from vllm.v1.worker.replayssm_utils import may_cross_mamba_block_boundary - for computed in (0, 12, 15, 16, 31): - for scheduled in (1, 4, 8): - may_cross = may_cross_mamba_block_boundary( - np.array([computed]), np.array([scheduled]), 16 - ) - for accepted in range(1, scheduled + 1): - crosses = computed // 16 != (computed + accepted) // 16 - assert may_cross or not crosses assert not may_cross_mamba_block_boundary(np.array([16]), np.array([8]), 16) assert may_cross_mamba_block_boundary(np.array([15]), np.array([1]), 16) + # Four accepted tokens end at the boundary. All eight end after it. + assert may_cross_mamba_block_boundary(np.array([12]), np.array([8]), 16) diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index 63e913bc7ae1..ba410e0dde16 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -556,27 +556,70 @@ def _make_gpu_ctx( ) -def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies(): +@pytest.mark.parametrize( + "mamba_type,wrapped", + [ + (MambaAttentionBackendEnum.MAMBA2, False), + (MambaAttentionBackendEnum.GDN_ATTN, False), + (MambaAttentionBackendEnum.GDN_ATTN, True), + ], +) +def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies(mamba_type, wrapped): mixer = _TestReplaySSMMixer() + mixer.replayssm_buffer_len = 16 + mixer.replayssm_executed_width = 8 + mixer.A = torch.ones(2) + is_gdn = mamba_type == MambaAttentionBackendEnum.GDN_ATTN + ring_len = 32 if is_gdn else 20 + shapes = ( + (2, 2), + (2, 2, 2), + (2, ring_len, 2), + (1, ring_len, 2) if is_gdn else (2, ring_len), + (2, ring_len) if is_gdn else (1, ring_len, 2), + ) + mixer.kv_cache = tuple(torch.zeros(3, *shape) for shape in shapes) + layer_name = "layers.0.mixer" + forward_context = {layer_name: mixer} spec = MambaSpec( block_size=16, - shapes=mixer.get_state_shape(), + shapes=shapes, dtypes=mixer.get_state_dtype(), + mamba_type=mamba_type, ) kv_config = KVCacheConfig( num_blocks=3, kv_cache_tensors=[], - kv_cache_groups=[KVCacheGroupSpec(["mixer"], spec)], + kv_cache_groups=[ + KVCacheGroupSpec( + [layer_name], + UniformTypeKVCacheSpecs( + block_size=16, kv_cache_specs={layer_name: spec} + ) + if wrapped + else spec, + ) + ], ) cfg = _TestConfig(num_layers=1, max_num_reqs=2) device = torch.device("cpu") canonical = _make_copy_bufs(cfg, kv_config, device) - replay = _make_gpu_ctx(cfg, kv_config, device, {"mixer": mixer}) + replay = _make_gpu_ctx(cfg, kv_config, device, forward_context) + initialize_replayssm_ring_trackers(kv_config, forward_context) + replay.initialize_from_forward_context( + kv_config, + forward_context, + _COPY_FUNCS, + [torch.zeros(cfg.max_num_reqs, 2, dtype=torch.int32)], + ) assert canonical.src_ptrs.cpu.numel() == cfg.max_num_reqs * 2 assert replay.num_states == 5 - assert replay.state_base_addrs.numel() == 5 - assert _COPY_FUNCS[MambaAttentionBackendEnum.MAMBA2] == _DEFAULT_COPY_FUNCS + assert replay.state_base_addrs.tolist() == [s.data_ptr() for s in mixer.kv_cache] + assert replay.state_conv_widths.tolist() == [2, 0, 0, 0, 0] + assert replay.replayssm is not None + assert replay.replayssm.mamba_type == mamba_type + assert _COPY_FUNCS[mamba_type] == _DEFAULT_COPY_FUNCS def test_replayssm_rejects_incompatible_ring_geometry_before_materialization(): diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 48deac0b4939..f997b915e6b4 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -326,10 +326,11 @@ def device_memory_utilization(self, value: float | None) -> None: use_replayssm: bool = False """Use a supported ReplaySSM decode kernel: cache recent recurrent inputs and skip the per-step full-state store, writing the checkpoint only on flush. - Triton and FlashInfer support the 'none' and 'align' Mamba cache modes. - Mamba2 speculative decoding requires FlashInfer. Pipeline parallelism is not - supported. Flushes are most efficient when mamba_block_size is a - multiple of replayssm_buffer_len, but this is not required.""" + Mamba2 supports Triton and FlashInfer in 'none' and 'align' cache modes. + Mamba2 speculative decoding and Qwen3.5 Gated DeltaNet require FlashInfer. + Pipeline parallelism is not supported. Flushes are most efficient when + mamba_block_size is a multiple of replayssm_buffer_len, but this is not + required.""" use_kda_recoverssm: bool = field(default=False, init=False) """Whether Kimi-K3 KDA uses RecoverSSM speculative decode.""" diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 891ab4e175a8..6ccbbbd680cf 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -4,9 +4,8 @@ import math from dataclasses import dataclass, replace -from typing import Literal, TypeVar +from typing import Literal -import numpy as np import torch from vllm._aiter_ops import rocm_aiter_ops @@ -92,23 +91,19 @@ def supports_device_cpu_query_lens_mismatch(cls) -> bool: return True -ArrayT = TypeVar("ArrayT", np.ndarray, torch.Tensor) - - def gdn_replayssm_prefill_rows( *, - is_prefilling: ArrayT, - is_last_prefill: ArrayT, - has_prior_state: ArrayT, - query_lens: ArrayT, - num_draft_tokens: ArrayT | None, -) -> ArrayT: + is_prefilling: torch.Tensor, + is_last_prefill: torch.Tensor, + has_prior_state: torch.Tensor, + query_lens: torch.Tensor, + num_draft_tokens: torch.Tensor | None, +) -> torch.Tensor: """Return the rows that GDN ReplaySSM must run as canonical prefill. Prompt chunks run as prefill, except the last chunk of a prompt that already has state when it is shaped like a decode row: one token, or one - token plus its draft tokens. That chunk replays instead. Accepts NumPy - arrays or CPU tensors. + token plus its draft tokens. That chunk replays instead. Inputs are CPU tensors. """ decode_shaped = query_lens == 1 if num_draft_tokens is not None: diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 916bdb87e1a7..d5aab7555726 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -707,13 +707,17 @@ def _get_mamba_gpu_state_copy_funcs( state_copy_funcs = copy_funcs[spec.mamba_type] if isinstance(layer, MambaBase) and layer.use_flashinfer_replayssm: if ( - spec.mamba_type != MambaAttentionBackendEnum.MAMBA2 + spec.mamba_type + not in ( + MambaAttentionBackendEnum.MAMBA2, + MambaAttentionBackendEnum.GDN_ATTN, + ) or len(spec.shapes) != 5 or len(spec.dtypes) != 5 or state_copy_funcs != (get_conv_copy_spec, get_temporal_copy_spec) ): raise ValueError( - "FlashInfer ReplaySSM requires two canonical Mamba2 " + "FlashInfer ReplaySSM requires two canonical Mamba2 or GDN " "states and three ring states" ) state_copy_funcs += (get_temporal_copy_spec,) * 3 @@ -953,7 +957,8 @@ def _populate_metadata( if is_flashinfer_replayssm: if len(kv_caches) != len(state_copy_funcs): raise ValueError( - "FlashInfer ReplaySSM requires five bound Mamba2 states" + "FlashInfer ReplaySSM requires five bound Mamba2 or GDN " + "states" ) mixers.append(attention) if len(kv_caches) < len(state_copy_funcs): @@ -1041,12 +1046,11 @@ def _populate_metadata( idx += 1 if mixers: - spec = kv_cache_group.kv_cache_spec - if not isinstance(spec, MambaSpec): - raise TypeError( - "FlashInfer ReplaySSM layers require a Mamba cache spec; " - f"got {type(spec).__name__}" - ) + spec = _get_mamba_spec_for_layer(kv_cache_group, layer_names[0]) + assert all( + _get_mamba_spec_for_layer(kv_cache_group, name) == spec + for name in layer_names + ), "FlashInfer ReplaySSM requires equal Mamba cache specs" replayssm_groups.append( (mixers, spec, aligned_index_block_tables[group_local_idx]) ) From aefb6c963e45bb24fb57886ee55e3243cd4c5cdd Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Mon, 9 Nov 2026 14:47:36 +0100 Subject: [PATCH 11/14] [Mamba] Use single backticks in GDN ReplaySSM docstrings Apply the check-docstring-style hook from main to the docstrings this PR adds. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Andrii Skliar --- vllm/model_executor/layers/mamba/gdn/base.py | 2 +- vllm/model_executor/layers/mamba/gdn/replayssm.py | 12 ++++++------ vllm/model_executor/layers/mamba/mamba_utils.py | 6 +++--- vllm/v1/worker/replayssm_utils.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 2910eb59abff..43ba7c0572a6 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -71,7 +71,7 @@ def mamba_type(self) -> MambaAttentionBackendEnum: @property def replayssm_rings(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Bound ``(u, k, G)`` rings, stored after the conv and SSM states.""" + """Bound `(u, k, G)` rings, stored after the conv and SSM states.""" u_cache, k_cache, g_cache = self.kv_cache[2:5] return u_cache, k_cache, g_cache diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py index fddbd36a95d8..ff2c7d325b98 100644 --- a/vllm/model_executor/layers/mamba/gdn/replayssm.py +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -60,13 +60,13 @@ def pack_replayssm_rows( uniform: bool = False, allow_strided_rows: bool = False, ) -> torch.Tensor: - """Pack variable-length token rows into ``[num_rows, executed_width, ...]``. + """Pack variable-length token rows into `[num_rows, executed_width, ...]`. - Row ``i`` holds the tokens in ``query_start_loc[i]:query_start_loc[i + 1]`` - followed by ``padding_value``. With ``uniform``, every row has exactly - ``executed_width`` tokens, so the result is a view of ``values`` when it + Row `i` holds the tokens in `query_start_loc[i]:query_start_loc[i + 1]` + followed by `padding_value`. With `uniform`, every row has exactly + `executed_width` tokens, so the result is a view of `values` when it is contiguous and aligned for CuTe, and a contiguous copy otherwise. - ``allow_strided_rows`` also preserves aligned, non-overlapping 2D rows. + `allow_strided_rows` also preserves aligned, non-overlapping 2D rows. """ num_rows = query_start_loc.numel() - 1 if uniform and values.shape[0] == num_rows * executed_width: @@ -129,7 +129,7 @@ def run_gdn_replayssm( ) -> torch.Tensor: """Run one FlashInfer decode step over the replay rows. - Returns the outputs of real tokens in ``output_indices`` order. The kernel + Returns the outputs of real tokens in `output_indices` order. The kernel reads the committed cursors but does not publish them; vLLM commits the accepted tokens once per model step, after every layer runs. """ diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 25e8913a5933..5d3c7eaf8134 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -57,7 +57,7 @@ def gdn_replayssm_executed_width(num_speculative_tokens: int) -> int: def configure_flashinfer_gdn_env() -> None: """Select FP16 state in FlashInfer's GDN prefix materializer. - FlashInfer reads ``GDN_UCACHE_STATE_DTYPE`` once, when ``flashinfer`` is + FlashInfer reads `GDN_UCACHE_STATE_DTYPE` once, when `flashinfer` is first imported, so it must be set before vLLM starts its workers. Raises: @@ -186,7 +186,7 @@ def gated_delta_net_replayssm_ring_dtypes( cls, model_dtype: ModelDType | torch.dtype, ) -> tuple[torch.dtype, torch.dtype, torch.dtype]: - """Return FlashInfer GDN ``(u, k, G)`` ring dtypes.""" + """Return FlashInfer GDN `(u, k, G)` ring dtypes.""" activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype) return (activation_dtype, activation_dtype, torch.float32) @@ -362,7 +362,7 @@ def gated_delta_net_replayssm_ring_shapes( head_k_dim: int, head_v_dim: int, ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - """Return FlashInfer GDN ``(u, k, G)`` ring shapes.""" + """Return FlashInfer GDN `(u, k, G)` ring shapes.""" local_k_heads = divide(num_k_heads, tp_world_size) local_v_heads = divide(num_v_heads, tp_world_size) slots = GDN_REPLAYSSM_RING_SLOTS diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index f340f8e63943..8725ac86a5a9 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -27,7 +27,7 @@ def may_cross_mamba_block_boundary( ) -> bool: """Whether any request could cross a Mamba block boundary this step. - A request accepts at most its scheduled tokens, so ``False`` means no prefix + A request accepts at most its scheduled tokens, so `False` means no prefix snapshot can be due and postprocess may skip materialization. """ return bool(((computed + scheduled) // block_size > computed // block_size).any()) From d642d839e71617e29ffa215b424dcceda93bca3a Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Mon, 9 Nov 2026 18:52:44 +0100 Subject: [PATCH 12/14] Add FlashInfer GDN ReplaySSM support for Qwen4 Preserve canonical PLE state and disable ReplaySSM for the attention-only MTP draft. Resolve wrapped layer specs during warmup. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/automatic_prefix_caching.md | 10 +- tests/test_config.py | 103 +++++++++++++++++- tests/v1/worker/test_mamba_utils.py | 63 +++++++++-- vllm/config/cache.py | 7 +- vllm/config/vllm.py | 9 ++ .../model_executor/warmup/replayssm_warmup.py | 15 +-- vllm/models/qwen4_exp/nvidia/model.py | 41 +++++-- vllm/models/qwen4_exp/nvidia/mtp.py | 2 + vllm/v1/worker/mamba_utils.py | 12 +- 9 files changed, 218 insertions(+), 44 deletions(-) diff --git a/docs/features/automatic_prefix_caching.md b/docs/features/automatic_prefix_caching.md index 7d30c3c0be83..c08a93b0260b 100644 --- a/docs/features/automatic_prefix_caching.md +++ b/docs/features/automatic_prefix_caching.md @@ -65,9 +65,9 @@ FlashInfer ReplaySSM has these limits: Prefix reuse still requires a shared prefix with a saved Mamba checkpoint. The matching rules for hybrid Mamba models above also apply. -#### Qwen3.5 Gated DeltaNet +#### Qwen3.5 and Qwen4Exp Gated DeltaNet -On Qwen3.5 models, `--use-replayssm` applies ReplaySSM to the Gated DeltaNet layers. It runs FlashInfer's UMMA kernel, which requires FlashInfer 0.7.3 or later, an SM100-class GPU, and FP16 recurrent state: +On Qwen3.5 and Qwen4Exp models, `--use-replayssm` applies ReplaySSM to the Gated DeltaNet layers. It runs FlashInfer's UMMA kernel, which requires FlashInfer 0.7.3 or later, an SM100-class GPU, and FP16 recurrent state: ```bash vllm serve nvidia/Qwen3.5-122B-A10B-NVFP4 \ @@ -78,13 +78,17 @@ vllm serve nvidia/Qwen3.5-122B-A10B-NVFP4 \ --speculative-config '{"method":"mtp","num_speculative_tokens":7}' ``` -Remove `--speculative-config` for standard decoding. Qwen3.5 Gated DeltaNet ReplaySSM has these limits: +Remove `--speculative-config` for standard decoding. The Gated DeltaNet ReplaySSM path has these limits: - Model activations and the convolution state must be BF16, with 128-element key and value heads and at most four value heads per key head. - Speculative decoding supports up to seven draft tokens. Adaptive verification is not supported; set `enable_adaptive_verification` to `false`. - `--replayssm-buffer-len` must be 16, the default. - Mamba cache stochastic rounding, pipeline parallelism, microbatching, and KV connectors are not supported. +Qwen3.8-Flash-Next uses the Qwen4Exp architecture. Its PLE convolution cache keeps its canonical state and speculative input window. Only the GDN layers use replay rings. The Qwen4Exp MTP draft uses attention layers and keeps ReplaySSM disabled. + +`--gdn-prefill-backend` selects the prefill kernel independently. Triton prefill can run with FlashInfer ReplaySSM decode. + With prefix caching, vLLM sets `GDN_UCACHE_STATE_DTYPE=fp16` before starting its workers, because FlashInfer reads it when first imported. If your own code imports FlashInfer before creating the engine, set this variable before that import. ## Example workloads diff --git a/tests/test_config.py b/tests/test_config.py index 56bb9c6d3b67..b4675b5cf441 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,6 +41,7 @@ SpeculativeConfig, VllmConfig, WatermarkConfig, + replace, update_config, ) from vllm.config.cache import MambaCacheMode, MambaDType @@ -162,18 +163,31 @@ def _gdn_replayssm_config( model_path: Path, mode: MambaCacheMode, ssm_dtype: MambaDType = "float16", + architecture: str = "Qwen3_5ForCausalLM", + use_replayssm: bool = True, + num_speculative_tokens: int = 0, ) -> VllmConfig: - from transformers import Qwen3_5TextConfig + from transformers import Qwen3_5TextConfig, Qwen4ExpConfig, Qwen4ExpTextConfig - Qwen3_5TextConfig( - architectures=["Qwen3_5ForCausalLM"], + is_qwen4 = architecture.startswith("Qwen4Exp") + config_cls = Qwen4ExpTextConfig if is_qwen4 else Qwen3_5TextConfig + hf_config = config_cls( + architectures=[architecture], num_hidden_layers=2, linear_key_head_dim=128, linear_value_head_dim=128, layer_types=["linear_attention", "full_attention"], max_position_embeddings=128, - ).save_pretrained(model_path) - return VllmConfig( + **( + {"linear_num_value_heads": 48, "ple_layer_ids": [1], "eos_token_id": 1} + if is_qwen4 + else {} + ), + ) + if architecture == "Qwen4ExpForConditionalGeneration": + hf_config = Qwen4ExpConfig(architectures=[architecture], text_config=hf_config) + hf_config.save_pretrained(model_path) + config = VllmConfig( model_config=ModelConfig( model=str(model_path), tokenizer_mode="skip", @@ -182,7 +196,7 @@ def _gdn_replayssm_config( enforce_eager=True, ), cache_config=CacheConfig( - use_replayssm=True, + use_replayssm=use_replayssm, replayssm_buffer_len=16, mamba_cache_mode=mode, enable_prefix_caching=mode == "align", @@ -193,6 +207,18 @@ def _gdn_replayssm_config( parallel_config=ParallelConfig(distributed_executor_backend="uni"), device_config=DeviceConfig(device="cpu"), ) + if num_speculative_tokens: + config = replace( + config, + speculative_config=SpeculativeConfig( + method="mtp", + num_speculative_tokens=num_speculative_tokens, + enable_adaptive_verification=False, + target_model_config=config.model_config, + target_parallel_config=config.parallel_config, + ), + ) + return config def test_gdn_replayssm_requires_fp16_state(tmp_path): @@ -200,6 +226,71 @@ def test_gdn_replayssm_requires_fp16_state(tmp_path): _gdn_replayssm_config(tmp_path, "none", "bfloat16") +@pytest.mark.parametrize( + "architecture,num_speculative_tokens", + [("Qwen4ExpForCausalLM", 0), ("Qwen4ExpForConditionalGeneration", 7)], +) +def test_qwen4_replayssm_preserves_ple_cache( + tmp_path, architecture, num_speculative_tokens +): + from vllm.models.qwen4_exp.nvidia.model import ( + Qwen4ExpForCausalLM, + Qwen4ExpForConditionalGeneration, + ) + + model_cls = ( + Qwen4ExpForCausalLM + if architecture == "Qwen4ExpForCausalLM" + else Qwen4ExpForConditionalGeneration + ) + baseline = _gdn_replayssm_config( + tmp_path, + "none", + architecture=architecture, + use_replayssm=False, + num_speculative_tokens=num_speculative_tokens, + ) + if current_platform.is_rocm(): + with pytest.raises(ValueError, match="does not support Qwen4Exp on ROCm"): + _gdn_replayssm_config(tmp_path, "none", architecture=architecture) + return + replay = _gdn_replayssm_config( + tmp_path, + "none", + architecture=architecture, + num_speculative_tokens=num_speculative_tokens, + ) + assert replay.model_config.supports_replayssm + assert replay.is_gdn_replayssm_enabled() + assert replay.use_flashinfer_replayssm + assert not baseline.is_gdn_replayssm_enabled() + baseline_gdn, baseline_ple = model_cls.get_mamba_specs_from_config(baseline) + replay_gdn, replay_ple = model_cls.get_mamba_specs_from_config(replay) + assert replay_gdn.shapes == baseline_gdn.shapes + ( + (48, 32, 128), + (16, 32, 128), + (48, 32), + ) + assert replay_gdn.dtypes == baseline_gdn.dtypes + ( + torch.bfloat16, + torch.bfloat16, + torch.float32, + ) + assert model_cls.get_mamba_state_shape_from_config(replay) == replay_gdn.shapes + assert model_cls.get_mamba_state_dtype_from_config(replay) == replay_gdn.dtypes + assert replay_gdn.page_size_bytes > baseline_gdn.page_size_bytes + assert replay_ple == baseline_ple + assert len(replay_ple.shapes) == 1 + assert replay_ple.tp_replicated + if num_speculative_tokens: + from vllm.models.qwen4_exp.nvidia.mtp import _make_draft_vllm_config + + draft = _make_draft_vllm_config(replay, 2, 1) + assert draft.model_config.architecture == "Qwen4ExpMTP" + assert not draft.cache_config.use_replayssm + assert replay.cache_config.use_replayssm + + def test_gdn_replayssm_align_configures_flashinfer_materializer(tmp_path, monkeypatch): """--use-replayssm selects FlashInfer's FP16 materializer before import.""" # Register the variable so monkeypatch removes what the validator sets. diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index aab73805f9b1..d7964c676f40 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -557,14 +557,22 @@ def _make_gpu_ctx( @pytest.mark.parametrize( - "mamba_type,wrapped", + "mamba_type,wrapped,baseline_type", [ - (MambaAttentionBackendEnum.MAMBA2, False), - (MambaAttentionBackendEnum.GDN_ATTN, False), - (MambaAttentionBackendEnum.GDN_ATTN, True), + (MambaAttentionBackendEnum.MAMBA2, False, None), + (MambaAttentionBackendEnum.GDN_ATTN, False, None), + (MambaAttentionBackendEnum.GDN_ATTN, True, None), + ( + MambaAttentionBackendEnum.GDN_ATTN, + True, + MambaAttentionBackendEnum.SHORT_CONV, + ), + (MambaAttentionBackendEnum.GDN_ATTN, False, MambaAttentionBackendEnum.MAMBA2), ], ) -def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies(mamba_type, wrapped): +def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies( + mamba_type, wrapped, baseline_type +): mixer = _TestReplaySSMMixer() mixer.replayssm_buffer_len = 16 mixer.replayssm_executed_width = 8 @@ -601,22 +609,57 @@ def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies(mamba_type, wr ) ], ) + states = list(mixer.kv_cache) + conv_widths = [2, 0, 0, 0, 0] + canonical_states = 2 + if baseline_type is not None: + baseline = _TestReplaySSMMixer() + baseline.use_replayssm = False + baseline.use_flashinfer_replayssm = False + baseline_shapes = shapes[: len(_COPY_FUNCS[baseline_type])] + baseline.kv_cache = tuple(torch.zeros(3, *shape) for shape in baseline_shapes) + forward_context["layers.1.baseline"] = baseline + kv_config.kv_cache_groups.append( + KVCacheGroupSpec( + ["layers.1.baseline"], + MambaSpec( + block_size=16, + shapes=baseline_shapes, + dtypes=(torch.float32,) * len(baseline_shapes), + mamba_type=baseline_type, + tp_replicated=baseline_type == MambaAttentionBackendEnum.SHORT_CONV, + ), + ) + ) + states.extend(baseline.kv_cache) + conv_widths.extend([2] if len(baseline_shapes) == 1 else [2, 0]) + canonical_states += len(baseline_shapes) cfg = _TestConfig(num_layers=1, max_num_reqs=2) device = torch.device("cpu") canonical = _make_copy_bufs(cfg, kv_config, device) replay = _make_gpu_ctx(cfg, kv_config, device, forward_context) initialize_replayssm_ring_trackers(kv_config, forward_context) + block_tables = [ + torch.zeros(cfg.max_num_reqs, 2, dtype=torch.int32) + for _ in kv_config.kv_cache_groups + ] + if baseline_type == MambaAttentionBackendEnum.MAMBA2: + with pytest.raises(ValueError, match="baseline Mamba"): + replay.initialize_from_forward_context( + kv_config, forward_context, _COPY_FUNCS, block_tables + ) + return replay.initialize_from_forward_context( kv_config, forward_context, _COPY_FUNCS, - [torch.zeros(cfg.max_num_reqs, 2, dtype=torch.int32)], + block_tables, ) - assert canonical.src_ptrs.cpu.numel() == cfg.max_num_reqs * 2 - assert replay.num_states == 5 - assert replay.state_base_addrs.tolist() == [s.data_ptr() for s in mixer.kv_cache] - assert replay.state_conv_widths.tolist() == [2, 0, 0, 0, 0] + assert canonical.src_ptrs.cpu.numel() == cfg.max_num_reqs * canonical_states + assert replay.num_states == len(states) + assert replay.state_base_addrs.tolist() == [s.data_ptr() for s in states] + assert replay.state_conv_widths.tolist() == conv_widths assert replay.replayssm is not None assert replay.replayssm.mamba_type == mamba_type assert _COPY_FUNCS[mamba_type] == _DEFAULT_COPY_FUNCS diff --git a/vllm/config/cache.py b/vllm/config/cache.py index f4513638601f..4bf0b011f6ec 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -321,13 +321,14 @@ def device_memory_utilization(self, value: float | None) -> None: replayssm_buffer_len: int = Field(default=16, gt=0) """ReplaySSM logical history length B. Mamba2 Triton uses B physical rows and Mamba2 FlashInfer uses B+T, where T is the verification length. Qwen3.5 - Gated DeltaNet requires B=16 and uses 32 physical rows. Kimi-K3 speculative - decode does not use B. Default 16.""" + and Qwen4Exp Gated DeltaNet require B=16 and use 32 physical rows. + Kimi-K3 speculative decode does not use B. Default 16.""" use_replayssm: bool = False """Use a supported ReplaySSM decode kernel: cache recent recurrent inputs and skip the per-step full-state store, writing the checkpoint only on flush. Mamba2 supports Triton and FlashInfer in 'none' and 'align' cache modes. - Mamba2 speculative decoding and Qwen3.5 Gated DeltaNet require FlashInfer. + Mamba2 speculative decoding, Qwen3.5 Gated DeltaNet, and Qwen4Exp Gated + DeltaNet require FlashInfer. Qwen4Exp ReplaySSM does not support ROCm. Pipeline parallelism is not supported. Flushes are most efficient when mamba_block_size is a multiple of replayssm_buffer_len, but this is not required.""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index f1972f55a540..e7390cc88d02 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -141,6 +141,8 @@ "Qwen3_5MoeForCausalLM", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", } ) @@ -3609,6 +3611,13 @@ def validate_mamba_block_size(self) -> "VllmConfig": def validate_gdn_replayssm(self) -> "VllmConfig": if not _uses_gdn_replayssm(self.cache_config, self.model_config): return self + if ( + self.model_config.architecture.startswith("Qwen4Exp") + and current_platform.is_rocm() + ): + raise ValueError( + "FlashInfer GDN ReplaySSM does not support Qwen4Exp on ROCm" + ) cache_config = self.cache_config if not 0 <= self.num_speculative_tokens <= 7: raise ValueError( diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index 56c317f9939d..17d35c4cdf42 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -110,16 +110,13 @@ def replayssm_lifecycle_warmup(runner: "GPUModelRunner") -> None: return from vllm.model_executor.layers.mamba.abstract import MambaBase - from vllm.v1.kv_cache_interface import MambaSpec + from vllm.v1.worker.mamba_utils import _get_mamba_spec_for_layer from vllm.v1.worker.replayssm_utils import ReplaySSMModelContext grouped = [] for group_id, group in enumerate(runner.kv_cache_config.kv_cache_groups): - spec = group.kv_cache_spec - if not isinstance(spec, MambaSpec): - continue - mixers = [ - layer + names = [ + name for name in group.layer_names if isinstance( layer := config.compilation_config.static_forward_context[name], @@ -127,8 +124,12 @@ def replayssm_lifecycle_warmup(runner: "GPUModelRunner") -> None: ) and layer.use_flashinfer_replayssm ] - if not mixers: + if not names: continue + spec = _get_mamba_spec_for_layer(group, names[0]) + mixers = [ + config.compilation_config.static_forward_context[name] for name in names + ] table = runner.block_tables.input_block_tables[group_id] # Match the runtime stride without changing its block-table contents. scratch_table = torch.empty_strided( diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index f705c9c76705..eb276691dcd5 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -34,12 +34,14 @@ from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsHCSP, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from vllm.model_executor.models.qwen3_5 import ( @@ -722,6 +724,7 @@ class Qwen4ExpForCausalLM( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, Qwen4ExpMixtureOfExperts, IsHybrid, ): @@ -844,17 +847,24 @@ def get_ple_mamba_state_shape_from_config( @classmethod def get_gdn_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) ) + if vllm_config.is_gdn_replayssm_enabled(): + dtypes += MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + vllm_config.model_config.dtype + ) + return dtypes @classmethod def get_gdn_mamba_state_shape_from_config( cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: parallel_config = vllm_config.parallel_config hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size @@ -863,7 +873,7 @@ def get_gdn_mamba_state_shape_from_config( if vllm_config.speculative_config else 0 ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, @@ -872,18 +882,27 @@ def get_gdn_mamba_state_shape_from_config( hf_config.linear_conv_kernel_dim, num_spec, ) + if vllm_config.is_gdn_replayssm_enabled(): + shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + ) + return shapes @classmethod def get_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: + ) -> tuple[torch.dtype, ...]: return cls.get_gdn_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: return cls.get_gdn_mamba_state_shape_from_config(vllm_config) @classmethod @@ -1160,14 +1179,14 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: + ) -> tuple[torch.dtype, ...]: return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config) @classmethod diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 45e576311533..a2eef680c3c1 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -171,6 +171,8 @@ def _make_draft_vllm_config( draft_vllm_config = replace( vllm_config, model_config=speculative_config.draft_model_config, + # Draft layers use attention, without GDN or PLE replay state. + cache_config=replace(vllm_config.cache_config, use_replayssm=False), ) # VllmConfig post-init derives the target quant config, so restore the # independently resolved draft quant config after replacement. diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 4d0c1c0a25cd..e71e22ed3e4a 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -937,7 +937,7 @@ def _populate_metadata( f"expected {self.num_groups} block tables, got {len(block_tables)}" ) replayssm_groups: list[tuple[list[Any], MambaSpec, torch.Tensor]] = [] - has_baseline_layer = False + has_baseline_temporal_state = False for group_local_idx, mamba_group_id in enumerate(self.mamba_group_ids): kv_cache_group = kv_cache_config.kv_cache_groups[mamba_group_id] mixers: list[Any] = [] @@ -953,7 +953,10 @@ def _populate_metadata( isinstance(attention, MambaBase) and attention.use_flashinfer_replayssm ) - has_baseline_layer |= not is_flashinfer_replayssm + has_baseline_temporal_state |= ( + not is_flashinfer_replayssm + and get_temporal_copy_spec in state_copy_funcs + ) if is_flashinfer_replayssm: if len(kv_caches) != len(state_copy_funcs): raise ValueError( @@ -1056,9 +1059,10 @@ def _populate_metadata( ) assert idx == self.num_states - if replayssm_groups and has_baseline_layer: + if replayssm_groups and has_baseline_temporal_state: raise ValueError( - "mixed FlashInfer ReplaySSM and baseline Mamba layers are unsupported" + "mixed FlashInfer ReplaySSM and baseline Mamba temporal states " + "are unsupported" ) # Cache per-group block-table base addresses and per-request stride. From 757aa94c37d91f28837b7a14ac87bfc858d71ed9 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Mon, 9 Nov 2026 19:26:26 +0100 Subject: [PATCH 13/14] Keep canonical PLE metadata outside Mamba ReplaySSM Select inherited replay metadata from the layer specification. This prevents Qwen4 PLE cache initialization from reading absent replay states. Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- .../test_replayssm_metadata_builder.py | 52 ++++++++++++++++++- vllm/v1/attention/backends/mamba_attn.py | 10 +++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/tests/v1/attention/test_replayssm_metadata_builder.py b/tests/v1/attention/test_replayssm_metadata_builder.py index 58f00d179a24..0817b3c21824 100644 --- a/tests/v1/attention/test_replayssm_metadata_builder.py +++ b/tests/v1/attention/test_replayssm_metadata_builder.py @@ -18,15 +18,63 @@ create_common_attn_metadata, create_vllm_config, ) -from vllm.config import SpeculativeConfig +from vllm.config import CacheConfig, CompilationConfig, SpeculativeConfig, VllmConfig from vllm.config.compilation import CUDAGraphMode -from vllm.config.mamba import MambaBackendEnum +from vllm.config.mamba import MambaBackendEnum, MambaConfig +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionMetadataBuilder, +) from vllm.v1.kv_cache_interface import MambaSpec BLOCK_SIZE = 16 DEVICE = torch.device("cpu") +@pytest.mark.parametrize( + "mamba_backend", [MambaBackendEnum.TRITON, MambaBackendEnum.FLASHINFER] +) +def test_ple_metadata_stays_canonical_with_replayssm_enabled(mamba_backend): + config = VllmConfig( + cache_config=CacheConfig(use_replayssm=True), + mamba_config=MambaConfig(backend=mamba_backend), + compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL), + ) + builder = PleShortConvAttentionMetadataBuilder( + MambaSpec( + block_size=BLOCK_SIZE, + shapes=((16, 64),), + dtypes=(torch.float32,), + mamba_type=MambaAttentionBackendEnum.SHORT_CONV, + ), + ["ple"], + config, + DEVICE, + ) + assert not builder.use_replayssm + assert not builder.use_flashinfer_replayssm + common = create_common_attn_metadata( + BatchSpec(seq_lens=[17, 18], query_lens=[1, 1]), + BLOCK_SIZE, + DEVICE, + arange_block_indices=True, + ).replace( + is_prefilling=torch.tensor([False, False], dtype=torch.bool), + ) + for metadata in ( + builder.build(0, common), + builder.build_for_cudagraph_capture(common), + ): + torch.testing.assert_close( + metadata.state_indices_tensor, common.block_table_tensor[:, 0] + ) + assert metadata.write_pos_d is None + assert metadata.is_flush_d is None + assert metadata.bc_pre_scratch is None + assert metadata.replayssm_scratch is None + assert metadata.replayssm_state_indices_d is None + + @dataclass class ReplaySSMBuildCase: seq_lens: list[int] diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 1bace4dab7b5..d454667ec459 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -15,6 +15,7 @@ AttentionMetadataBuilder, CommonAttentionMetadata, ) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.utils import ( NULL_BLOCK_ID, compute_causal_conv1d_metadata, @@ -107,9 +108,14 @@ def __init__( self.compilation_config = vllm_config.compilation_config self.num_spec_tokens: int = vllm_config.num_speculative_tokens self.use_spec_decode = self.num_spec_tokens > 0 - self.use_replayssm = vllm_config.cache_config.use_replayssm + self.use_replayssm = ( + vllm_config.cache_config.use_replayssm + and kv_cache_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2 + ) self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len - self.use_flashinfer_replayssm = vllm_config.use_flashinfer_replayssm + self.use_flashinfer_replayssm = ( + self.use_replayssm and vllm_config.use_flashinfer_replayssm + ) scheduler_config = vllm_config.scheduler_config self.decode_cudagraph_max_bs: int = scheduler_config.max_num_seqs From ef69cf27381f62c44fa1576379ea0b6496f44027 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Mon, 9 Nov 2026 19:28:24 +0100 Subject: [PATCH 14/14] Document Qwen3.8 Flash Next MTP cache alignment Assisted-by: OpenAI Codex Signed-off-by: Andrii Skliar --- docs/features/automatic_prefix_caching.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/features/automatic_prefix_caching.md b/docs/features/automatic_prefix_caching.md index c08a93b0260b..324d9653594c 100644 --- a/docs/features/automatic_prefix_caching.md +++ b/docs/features/automatic_prefix_caching.md @@ -87,6 +87,8 @@ Remove `--speculative-config` for standard decoding. The Gated DeltaNet ReplaySS Qwen3.8-Flash-Next uses the Qwen4Exp architecture. Its PLE convolution cache keeps its canonical state and speculative input window. Only the GDN layers use replay rings. The Qwen4Exp MTP draft uses attention layers and keeps ReplaySSM disabled. +For `Qwen/Qwen3.8-Flash-Next-FP8`, add `--tensor-parallel-size 4 --block-size 192` to the example. With seven MTP drafts, the explicit block size satisfies the QSA raw-key ring alignment. + `--gdn-prefill-backend` selects the prefill kernel independently. Triton prefill can run with FlashInfer ReplaySSM decode. With prefix caching, vLLM sets `GDN_UCACHE_STATE_DTYPE=fp16` before starting its workers, because FlashInfer reads it when first imported. If your own code imports FlashInfer before creating the engine, set this variable before that import.