diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 8d4b42331de4..69d94bafb43b 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -231,7 +231,11 @@ steps: - vllm/v1/worker/gpu_worker.py commands: - VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v -s v1/e2e/test_replayssm_decode.py - - VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v kernels/mamba/test_ssu_dispatch.py::test_replayssm_materializes_only_accepted_boundary_with_compacted_rows kernels/mamba/test_ssu_dispatch.py::test_replayssm_postprocess_commits_staged_transition + - >- + VLLM_TEST_REQUIRE_REPLAYSSM=1 pytest -v -s + kernels/mamba/test_ssu_dispatch.py::test_replayssm_materializes_only_accepted_boundary_with_compacted_rows + kernels/mamba/test_ssu_dispatch.py::test_replayssm_postprocess_commits_staged_transition + kernels/mamba/test_ssu_dispatch.py::test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers mirror: amd: automatic_shard: true diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 10b7b42a2445..b61036f416ab 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -787,6 +787,8 @@ steps: - vllm/platforms/cuda.py - vllm/cute_utils/ - vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/ + - vllm/model_executor/layers/mamba/gdn/ + - vllm/model_executor/layers/mamba/mamba_utils.py - vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_sm100.py - cmake/external_projects/fmha_sm100.cmake - vllm/models/minimax_m3/common/sparse_attention.py @@ -803,6 +805,7 @@ steps: - tests/kernels/quantization/nvfp4_utils.py - tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py - tests/kernels/mamba/test_gdn_prefill_cutedsl.py + - tests/kernels/mamba/test_gdn_replayssm.py - tests/kernels/test_bf16x3_router_gemm_cutedsl.py - tests/kernels/test_top_k_per_row.py - csrc/fused_mqa_topk/ @@ -842,6 +845,7 @@ steps: - pytest -v -s tests/kernels/moe/test_flashinfer.py - pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py - pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py + - pytest -v -s tests/kernels/mamba/test_gdn_replayssm.py mirror: amd: automatic_shard: true diff --git a/docs/features/automatic_prefix_caching.md b/docs/features/automatic_prefix_caching.md index 65135565f170..324d9653594c 100644 --- a/docs/features/automatic_prefix_caching.md +++ b/docs/features/automatic_prefix_caching.md @@ -65,6 +65,34 @@ FlashInfer ReplaySSM has these limits: Prefix reuse still requires a shared prefix with a saved Mamba checkpoint. The matching rules for hybrid Mamba models above also apply. +#### Qwen3.5 and Qwen4Exp Gated DeltaNet + +On Qwen3.5 and Qwen4Exp models, `--use-replayssm` applies ReplaySSM to the Gated DeltaNet layers. It runs FlashInfer's UMMA kernel, which requires FlashInfer 0.7.3 or later, an SM100-class GPU, and FP16 recurrent state: + +```bash +vllm serve nvidia/Qwen3.5-122B-A10B-NVFP4 \ + --dtype bfloat16 \ + --mamba-ssm-cache-dtype float16 \ + --use-replayssm \ + --enable-prefix-caching \ + --speculative-config '{"method":"mtp","num_speculative_tokens":7}' +``` + +Remove `--speculative-config` for standard decoding. The Gated DeltaNet ReplaySSM path has these limits: + +- Model activations and the convolution state must be BF16, with 128-element key and value heads and at most four value heads per key head. +- Speculative decoding supports up to seven draft tokens. Adaptive verification is not supported; set `enable_adaptive_verification` to `false`. +- `--replayssm-buffer-len` must be 16, the default. +- Mamba cache stochastic rounding, pipeline parallelism, microbatching, and KV connectors are not supported. + +Qwen3.8-Flash-Next uses the Qwen4Exp architecture. Its PLE convolution cache keeps its canonical state and speculative input window. Only the GDN layers use replay rings. The Qwen4Exp MTP draft uses attention layers and keeps ReplaySSM disabled. + +For `Qwen/Qwen3.8-Flash-Next-FP8`, add `--tensor-parallel-size 4 --block-size 192` to the example. With seven MTP drafts, the explicit block size satisfies the QSA raw-key ring alignment. + +`--gdn-prefill-backend` selects the prefill kernel independently. Triton prefill can run with FlashInfer ReplaySSM decode. + +With prefix caching, vLLM sets `GDN_UCACHE_STATE_DTYPE=fp16` before starting its workers, because FlashInfer reads it when first imported. If your own code imports FlashInfer before creating the engine, set this variable before that import. + ## Example workloads We describe two example workloads, where APC can provide huge performance benefit: diff --git a/tests/kernels/mamba/test_gdn_replayssm.py b/tests/kernels/mamba/test_gdn_replayssm.py new file mode 100644 index 000000000000..e159f0a06644 --- /dev/null +++ b/tests/kernels/mamba/test_gdn_replayssm.py @@ -0,0 +1,344 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +import sys +from types import ModuleType + +import pytest +import torch +import torch.nn.functional as F + +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_kernel, + load_gdn_replayssm_materializer, + pack_replayssm_rows, + run_gdn_replayssm, +) + + +@pytest.mark.parametrize("loaded_dtype", [torch.float16, torch.bfloat16]) +def test_materializer_requires_fp16_state(monkeypatch, loaded_dtype): + """A FlashInfer imported before vLLM set its state dtype must be rejected.""" + provider = ModuleType("provider") + materialize = lambda **kwargs: None + monkeypatch.setattr(provider, "gdn_prefix_materialize", materialize, raising=False) + monkeypatch.setattr(provider, "state_ty_torch", lambda: loaded_dtype, raising=False) + monkeypatch.setitem( + sys.modules, "flashinfer.gdn_kernels.gdn_prefix_materialize", provider + ) + load_gdn_replayssm_materializer.cache_clear() + try: + if loaded_dtype == torch.float16: + assert load_gdn_replayssm_materializer() is materialize + else: + with pytest.raises(ValueError, match="GDN_UCACHE_STATE_DTYPE=fp16"): + load_gdn_replayssm_materializer() + finally: + load_gdn_replayssm_materializer.cache_clear() + + +def test_pack_replayssm_rows_preserves_tokens_and_masks_padding(): + values = torch.tensor([[1.0], [2.0], [3.0]]) + packed = pack_replayssm_rows( + values, torch.tensor([0, 1, 3]), 4, padding_value=-torch.inf + ) + torch.testing.assert_close( + packed.squeeze(-1), + torch.tensor( + [ + [1.0, -torch.inf, -torch.inf, -torch.inf], + [2.0, 3.0, -torch.inf, -torch.inf], + ] + ), + ) + + +def test_uniform_replayssm_packing_preserves_alignment_and_padding(): + width = 4 + values = torch.arange(2 * width * 4, dtype=torch.bfloat16).view(2 * width, 4) + starts = torch.tensor([0, width, 2 * width]) + packed = pack_replayssm_rows(values, starts, width, uniform=True) + assert packed.data_ptr() == values.data_ptr() + torch.testing.assert_close(packed, pack_replayssm_rows(values, starts, width)) + # A gate view can be contiguous but start at an unaligned byte offset. + unaligned = torch.zeros(values.numel() + 1, dtype=values.dtype)[1:].view_as(values) + unaligned.copy_(values) + aligned = pack_replayssm_rows(unaligned, starts, width, uniform=True) + assert aligned.data_ptr() % 16 == 0 + torch.testing.assert_close(aligned, packed) + # A column slice of a wider projection is copied to a contiguous buffer. + projection = torch.zeros(2 * width, 8, dtype=values.dtype)[:, :4] + projection.copy_(values) + sliced = pack_replayssm_rows(projection, starts, width, uniform=True) + assert sliced.is_contiguous() and sliced.data_ptr() % 16 == 0 + torch.testing.assert_close(sliced, packed) + # Eager staging can include a zero-token graph-padding row. + padded_starts = torch.tensor([0, width, width]) + padded = pack_replayssm_rows(values[:width], padded_starts, width, uniform=True) + torch.testing.assert_close(padded[0], values[:width]) + assert padded[1].eq(0).all() + + +def test_uniform_replayssm_passes_qkv_views_and_aligned_gates(): + width = 4 + tokens, dim = 2 * width, 128 + projection = torch.arange(tokens * 4 * dim).to(torch.bfloat16).view(tokens, -1) + mixed_qkv = projection[:, : 3 * dim] + gates = torch.ones(tokens, 2, dtype=torch.bfloat16) + a, b = gates[:, :1], gates[:, 1:] + + def kernel(**kwargs): + # UMMA makes its own contiguous Q/K/V copies. Keep the packed view. + assert kwargs["q"].data_ptr() == mixed_qkv.data_ptr() + for name, expected in zip(("q", "k", "v"), mixed_qkv.split(dim, -1)): + torch.testing.assert_close( + kwargs[name].flatten(0, 2), expected, rtol=0, atol=0 + ) + for name, expected in (("a", a), ("b", b)): + assert kwargs[name].is_contiguous() + assert kwargs[name].data_ptr() % 16 == 0 + torch.testing.assert_close(kwargs[name].flatten(0, 1), expected) + kwargs["output"].copy_(kwargs["q"]) + + output = run_gdn_replayssm( + kernel=kernel, + executed_width=width, + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=torch.zeros(1), + dt_bias=torch.zeros(1), + checkpoint_state=torch.zeros(3, 1, dim, dim, dtype=torch.float16), + replayssm_cache=( + torch.zeros(3, 1, 32, dim, dtype=torch.bfloat16), + torch.zeros(3, 1, 32, dim, dtype=torch.bfloat16), + torch.zeros(3, 1, 32), + ), + ring_start=torch.zeros(3, dtype=torch.int32), + num_committed=torch.zeros(3, dtype=torch.int32), + query_start_loc=torch.tensor([0, width, tokens], dtype=torch.int32), + state_indices=torch.tensor([1, 2], dtype=torch.int32), + output_indices=torch.arange(tokens), + num_k_heads=1, + num_v_heads=1, + head_k_dim=dim, + head_v_dim=dim, + uniform_query_length=width, + ) + torch.testing.assert_close(output[:, 0], mixed_qkv[:, :dim], rtol=0, atol=0) + + +def _reference_gdn_step( + state, + q, + k, + v, + a, + b, + A_log, + dt_bias, +): + """FP32 recurrence using FlashInfer's effective BF16 gate constants.""" + state = state.float().clone() + A_log = A_log.to(torch.bfloat16).float() + dt_bias = dt_bias.to(torch.bfloat16).float() + group_size = v.shape[1] // k.shape[1] + outputs = [] + for token in range(q.shape[0]): + k_hv = F.normalize(k[token].float(), dim=-1).repeat_interleave( + group_size, dim=0 + ) + q_hv = F.normalize(q[token].float(), dim=-1).repeat_interleave( + group_size, dim=0 + ) + q_hv *= 1.0 / math.sqrt(q.shape[-1]) + decay = -torch.exp(A_log) * F.softplus(a[token].float() + dt_bias) + beta = torch.sigmoid(b[token].float()) + state *= torch.exp(decay)[:, None, None] + prediction = torch.einsum("hvk,hk->hv", state, k_hv) + update = (v[token].float() - prediction) * beta[:, None] + state += update[:, :, None] * k_hv[:, None, :] + outputs.append(torch.einsum("hvk,hk->hv", state, q_hv)) + return torch.stack(outputs), state + + +def _reconstruct_replayssm_state( + checkpoint, + k_cache, + u_cache, + g_cache, + committed, + ring_start, +): + state = checkpoint.float().clone() + if committed == 0: + return state + rows = torch.tensor( + [(ring_start + offset) % k_cache.size(1) for offset in range(committed)], + dtype=torch.long, + device=checkpoint.device, + ) + k_logical = k_cache.index_select(1, rows).float().repeat_interleave(2, dim=0) + u_logical = u_cache.index_select(1, rows).float() + g_logical = g_cache.index_select(1, rows).float() + final_g = g_logical[:, -1] + weights = torch.exp(final_g[:, None] - g_logical) + return torch.exp(final_g)[:, None, None] * state + torch.einsum( + "hpv,hpk->hvk", weights[:, :, None] * u_logical, k_logical + ) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="FlashInfer GDN ReplaySSM requires an SM100-class CUDA device", +) +@pytest.mark.parametrize( + "width,query_len,accepted", + [(1, 1, 1), (4, 3, 2), (8, 6, 4), (8, 8, 4)], +) +@torch.inference_mode() +def test_gdn_replayssm_output_and_accepted_state(width, query_len, accepted): + """Check nonzero outputs and accepted state through replay, wrap, and flush.""" + device, dtype = torch.device("cuda"), torch.bfloat16 + generator = torch.Generator(device=device).manual_seed(20260909 + width) + ring_slots = 32 + + def random_tensor(*shape): + return torch.randn(*shape, generator=generator, device=device).to(dtype) + + # Use one page stride for every state, as MambaBase.bind_kv_cache does. + shapes = [ + (2, 128, 128), + (2, ring_slots, 128), + (1, ring_slots, 128), + (2, ring_slots), + ] + dtypes = [torch.float16, dtype, dtype, torch.float32] + sizes = [math.prod(shape) * dt.itemsize for shape, dt in zip(shapes, dtypes)] + pages = torch.zeros(2, sum(sizes) + 16, dtype=torch.uint8, device=device) + states = [] + offset = 0 + for shape, dt, size in zip(shapes, dtypes, sizes): + states.append(pages[:, offset : offset + size].view(dt).view(2, *shape)) + offset += size + checkpoint, u_cache, k_cache, g_cache = states + pages[0].fill_(17) # Nonzero canaries catch writes to the padding page. + checkpoint.copy_(random_tensor(2, 2, 128, 128) * 0.03) + ring_start = torch.zeros(2, dtype=torch.int32, device=device) + if width == 1: + ring_start[1] = 24 # Cross the ring boundary before the first flush. + committed = torch.zeros_like(ring_start) + A_log = torch.nn.Parameter(torch.full((2,), -3.0, device=device)) + dt_bias = torch.nn.Parameter(random_tensor(2) * 0.2) + accepted_state = checkpoint[1].float().clone() + padding_page = pages[0].clone() + capture = width in (1, 4) + graph = None + + # Eighteen steps cross a flush and the ring boundary. + for step in range(18): + if capture and step == 9: + # Captured execution must observe an in-place weight reload. + A_log.add_(2.0) + dt_bias.add_(1.0) + q, k = random_tensor(query_len, 1, 128), random_tensor(query_len, 1, 128) + v = random_tensor(query_len, 2, 128) * 0.1 + a, b = random_tensor(query_len, 2) * 0.2, random_tensor(query_len, 2) + mixed_qkv = torch.cat((q.flatten(1), k.flatten(1), v.flatten(1)), dim=-1) + if width == query_len and width in (4, 8): + projection = mixed_qkv.new_zeros(query_len, mixed_qkv.shape[1] + 128) + projection[:, : mixed_qkv.shape[1]].copy_(mixed_qkv) + mixed_qkv = projection[:, : mixed_qkv.shape[1]] + gates = torch.stack((a, b), dim=1) + a, b = gates[:, 0], gates[:, 1] + if capture: + # Captured padding has input storage but no token or live state. + mixed_qkv, a, b = ( + torch.cat((values, torch.ones_like(values[:1]).expand(width, -1))) + for values in (mixed_qkv, a, b) + ) + inputs = dict( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=A_log, + dt_bias=dt_bias, + checkpoint_state=checkpoint, + replayssm_cache=(u_cache, k_cache, g_cache), + ring_start=ring_start, + num_committed=committed, + query_start_loc=torch.tensor( + [0, query_len, query_len] if capture else [0, query_len], + dtype=torch.int32, + device=device, + ), + state_indices=torch.tensor( + [1, 0] if capture else [1], dtype=torch.int32, device=device + ), + output_indices=torch.tensor( + list(range(query_len)) + ([width] if capture else []), + dtype=torch.int32, + device=device, + ), + kernel=load_gdn_replayssm_kernel(), + executed_width=width, + uniform_query_length=query_len, + num_k_heads=1, + num_v_heads=2, + head_k_dim=128, + head_v_dim=128, + ) + if capture: + if graph is None: + graph_inputs = inputs + saved_pages = pages.clone() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run_gdn_replayssm(**graph_inputs) + torch.cuda.current_stream().wait_stream(stream) + pages.copy_(saved_pages) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_output = run_gdn_replayssm(**graph_inputs) + pages.copy_(saved_pages) + else: + for name in ("mixed_qkv", "a", "b"): + graph_inputs[name].copy_(inputs[name]) + graph.replay() + output = graph_output[:query_len] + assert graph_output[query_len:].eq(0).all() + torch.testing.assert_close(pages[0], padding_page, rtol=0, atol=0) + a, b = a[:query_len], b[:query_len] + else: + output = run_gdn_replayssm(**inputs) + expected_output, _ = _reference_gdn_step( + accepted_state, q, k, v, a, b, A_log, dt_bias + ) + torch.testing.assert_close(output.float(), expected_output, atol=8e-3, rtol=0) + _, accepted_state = _reference_gdn_step( + accepted_state, + q[:accepted], + k[:accepted], + v[:accepted], + a[:accepted], + b[:accepted], + A_log, + dt_bias, + ) + old_committed = int(committed[1].item()) + if old_committed + width > 16: + ring_start[1] = (ring_start[1] + old_committed) % ring_slots + committed[1] = accepted + else: + committed[1] = old_committed + accepted + reconstructed = _reconstruct_replayssm_state( + checkpoint[1], + k_cache[1], + u_cache[1], + g_cache[1], + int(committed[1].item()), + int(ring_start[1].item()), + ) + torch.testing.assert_close(reconstructed, accepted_state, atol=2e-2, rtol=0) diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index f00f927f8356..d0f42e7948ec 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -315,19 +315,34 @@ def test_replayssm_materializes_only_accepted_boundary_with_compacted_rows( @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -@pytest.mark.parametrize("post_step", [False, True], ids=["v1", "v2"]) -def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_step): - """Skip prefill self-copies, reset trackers, and preserve in-place MTP replay.""" +@pytest.mark.parametrize( + "post_step,ring_slots,executed_width,query_len,committed,flush_count,ring_start", + [ + # Mamba2: 2 + 4 fits the window, so replay 2 old + 4 accepted tokens. + pytest.param(False, 20, 0, 4, 2, 6, 3, id="mamba2-pre-step"), + pytest.param(True, 20, 0, 4, 2, 6, 3, id="mamba2-post-step"), + # GDN standard decoding: 16 + 1 > 16 folds; publish only the new token. + pytest.param(True, 32, 1, 1, 16, 1, 19, id="gdn-stp-fold"), + # A 4-token row in a width-8 GDN launch folds at 10 + 8 > 16, although + # 10 + 4 would fit. + pytest.param(True, 32, 8, 4, 10, 4, 13, id="gdn-mtp-fold"), + pytest.param(True, 32, 8, 4, 8, 12, 3, id="gdn-mtp-append"), + ], +) +def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers( + post_step, ring_slots, executed_width, query_len, committed, flush_count, ring_start +): + """Skip prefill self-copies, reset trackers, and preserve in-place replay.""" mixer = _replayssm_mixer( kv_cache=( torch.empty(0, device="cuda"), torch.empty(6, 1, 1, 1, device="cuda"), - torch.empty(6, 1, 20, 1, device="cuda"), - torch.empty(6, 1, 20, device="cuda"), - torch.empty(6, 1, 20, 1, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), + torch.empty(6, 1, ring_slots, device="cuda"), + torch.empty(6, 1, ring_slots, 1, device="cuda"), ), ring_start=_cuda_ints([3] * 6), - committed=_cuda_ints([2] * 6), + committed=_cuda_ints([2, 2, 2, 2, committed, 2]), A=torch.empty(1, device="cuda"), ) spec = MambaSpec( @@ -345,13 +360,16 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st ] ) context = ReplaySSMModelContext.create([([mixer], spec, table)], max_num_reqs=4) - # Reordered rows: canonical prefill in slot 2; in-place MTP replay in slot 4; - # padded slot; padded request. + # Reordered rows: canonical prefill in slot 2; in-place replay in slot 4 + # that accepts every token and reaches the block boundary at 32; padded + # slot; padded request. _postprocess_replayssm_kernel[(4, 1)]( _cuda_ints([1, 2, 0, -1]), - _cuda_ints([0, 48, 52, 68, 68]) if post_step else _cuda_ints([48, 4, 16, 0]), - _cuda_ints([16, 48, 32]) if post_step else _cuda_ints([0, 0, 28]), - _cuda_ints([1, 1, 4]), + _cuda_ints([0, 48, 48 + query_len, 64 + query_len, 64 + query_len]) + if post_step + else _cuda_ints([48, query_len, 16, 0]), + _cuda_ints([16, 48, 32]) if post_step else _cuda_ints([0, 0, 32 - query_len]), + _cuda_ints([1, 1, query_len]), torch.tensor([True, False, True, True], device="cuda"), _cuda_ints([0, 2, 1]), context.block_table_ptrs, @@ -366,23 +384,26 @@ def test_replayssm_skips_canonical_self_copy_but_resets_prefill_trackers(post_st MAX_NUM_REQS=4, MAMBA_BLOCK_SIZE=16, LOGICAL_WINDOW=16, - RING_BUFFER_LEN=20, + RING_BUFFER_LEN=ring_slots, PAD_SLOT_ID=NULL_BLOCK_ID, QUERY_METADATA_IS_CUMULATIVE=post_step, NUM_COMPUTED_IS_POST_STEP=post_step, HAS_IDX_MAPPING=True, MATERIALIZE_PREFIXES=True, LIVE_COL_IS_ZERO=False, + EXECUTED_QUERY_WIDTH=executed_width, ) _compact_replayssm_requests_kernel[(1,)]( context.plan_flush_count, context.active_request_indices, + context.num_active, 4, MAX_NUM_REQS=4, BLOCK_SIZE=4, ) - assert context.plan_flush_count.tolist() == [-1, 6, -1, -1] + assert context.plan_flush_count.tolist() == [-1, flush_count, -1, -1] + assert context.plan_ring_start[1].item() == ring_start assert context.active_request_indices.tolist() == [1, -1, -1, -1] assert context.src_slots[0, 1].item() == 4 assert context.dst_slots[0, 1].item() == 4 diff --git a/tests/model_executor/test_replayssm_warmup.py b/tests/model_executor/test_replayssm_warmup.py index 0a04db9bf5d1..09fe7d2bb9a0 100644 --- a/tests/model_executor/test_replayssm_warmup.py +++ b/tests/model_executor/test_replayssm_warmup.py @@ -44,6 +44,7 @@ def _autotune_runner( use_flashinfer_replayssm=( use_replayssm and backend == MambaBackendEnum.FLASHINFER ), + is_gdn_replayssm_enabled=lambda: False, ), uniform_decode_query_len=query_len, decode_query_len=query_len, diff --git a/tests/test_config.py b/tests/test_config.py index 64808b61078e..b4675b5cf441 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,8 +41,10 @@ SpeculativeConfig, VllmConfig, WatermarkConfig, + replace, update_config, ) +from vllm.config.cache import MambaCacheMode, MambaDType from vllm.config.compilation import CompilationMode, CUDAGraphMode, PassConfig from vllm.config.kernel import IrOpPriorityConfig from vllm.config.load import LoadConfig @@ -157,6 +159,155 @@ def test_flashinfer_replayssm_validates_config( VllmConfig.validate_mamba_cached_kernel(config) +def _gdn_replayssm_config( + model_path: Path, + mode: MambaCacheMode, + ssm_dtype: MambaDType = "float16", + architecture: str = "Qwen3_5ForCausalLM", + use_replayssm: bool = True, + num_speculative_tokens: int = 0, +) -> VllmConfig: + from transformers import Qwen3_5TextConfig, Qwen4ExpConfig, Qwen4ExpTextConfig + + is_qwen4 = architecture.startswith("Qwen4Exp") + config_cls = Qwen4ExpTextConfig if is_qwen4 else Qwen3_5TextConfig + hf_config = config_cls( + architectures=[architecture], + num_hidden_layers=2, + linear_key_head_dim=128, + linear_value_head_dim=128, + layer_types=["linear_attention", "full_attention"], + max_position_embeddings=128, + **( + {"linear_num_value_heads": 48, "ple_layer_ids": [1], "eos_token_id": 1} + if is_qwen4 + else {} + ), + ) + if architecture == "Qwen4ExpForConditionalGeneration": + hf_config = Qwen4ExpConfig(architectures=[architecture], text_config=hf_config) + hf_config.save_pretrained(model_path) + config = VllmConfig( + model_config=ModelConfig( + model=str(model_path), + tokenizer_mode="skip", + dtype="bfloat16", + max_model_len=128, + enforce_eager=True, + ), + cache_config=CacheConfig( + use_replayssm=use_replayssm, + replayssm_buffer_len=16, + mamba_cache_mode=mode, + enable_prefix_caching=mode == "align", + mamba_cache_dtype="auto", + mamba_ssm_cache_dtype=ssm_dtype, + ), + scheduler_config=SchedulerConfig(max_model_len=128, is_encoder_decoder=False), + parallel_config=ParallelConfig(distributed_executor_backend="uni"), + device_config=DeviceConfig(device="cpu"), + ) + if num_speculative_tokens: + config = replace( + config, + speculative_config=SpeculativeConfig( + method="mtp", + num_speculative_tokens=num_speculative_tokens, + enable_adaptive_verification=False, + target_model_config=config.model_config, + target_parallel_config=config.parallel_config, + ), + ) + return config + + +def test_gdn_replayssm_requires_fp16_state(tmp_path): + with pytest.raises(ValueError, match="--mamba-ssm-cache-dtype float16"): + _gdn_replayssm_config(tmp_path, "none", "bfloat16") + + +@pytest.mark.parametrize( + "architecture,num_speculative_tokens", + [("Qwen4ExpForCausalLM", 0), ("Qwen4ExpForConditionalGeneration", 7)], +) +def test_qwen4_replayssm_preserves_ple_cache( + tmp_path, architecture, num_speculative_tokens +): + from vllm.models.qwen4_exp.nvidia.model import ( + Qwen4ExpForCausalLM, + Qwen4ExpForConditionalGeneration, + ) + + model_cls = ( + Qwen4ExpForCausalLM + if architecture == "Qwen4ExpForCausalLM" + else Qwen4ExpForConditionalGeneration + ) + baseline = _gdn_replayssm_config( + tmp_path, + "none", + architecture=architecture, + use_replayssm=False, + num_speculative_tokens=num_speculative_tokens, + ) + if current_platform.is_rocm(): + with pytest.raises(ValueError, match="does not support Qwen4Exp on ROCm"): + _gdn_replayssm_config(tmp_path, "none", architecture=architecture) + return + replay = _gdn_replayssm_config( + tmp_path, + "none", + architecture=architecture, + num_speculative_tokens=num_speculative_tokens, + ) + assert replay.model_config.supports_replayssm + assert replay.is_gdn_replayssm_enabled() + assert replay.use_flashinfer_replayssm + assert not baseline.is_gdn_replayssm_enabled() + baseline_gdn, baseline_ple = model_cls.get_mamba_specs_from_config(baseline) + replay_gdn, replay_ple = model_cls.get_mamba_specs_from_config(replay) + assert replay_gdn.shapes == baseline_gdn.shapes + ( + (48, 32, 128), + (16, 32, 128), + (48, 32), + ) + assert replay_gdn.dtypes == baseline_gdn.dtypes + ( + torch.bfloat16, + torch.bfloat16, + torch.float32, + ) + assert model_cls.get_mamba_state_shape_from_config(replay) == replay_gdn.shapes + assert model_cls.get_mamba_state_dtype_from_config(replay) == replay_gdn.dtypes + assert replay_gdn.page_size_bytes > baseline_gdn.page_size_bytes + assert replay_ple == baseline_ple + assert len(replay_ple.shapes) == 1 + assert replay_ple.tp_replicated + if num_speculative_tokens: + from vllm.models.qwen4_exp.nvidia.mtp import _make_draft_vllm_config + + draft = _make_draft_vllm_config(replay, 2, 1) + assert draft.model_config.architecture == "Qwen4ExpMTP" + assert not draft.cache_config.use_replayssm + assert replay.cache_config.use_replayssm + + +def test_gdn_replayssm_align_configures_flashinfer_materializer(tmp_path, monkeypatch): + """--use-replayssm selects FlashInfer's FP16 materializer before import.""" + # Register the variable so monkeypatch removes what the validator sets. + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "") + monkeypatch.delenv("GDN_UCACHE_STATE_DTYPE") + config = _gdn_replayssm_config(tmp_path, "none") + assert config.is_gdn_replayssm_enabled() + assert config.use_flashinfer_replayssm + assert "GDN_UCACHE_STATE_DTYPE" not in os.environ + config = _gdn_replayssm_config(tmp_path, "align") + assert config.is_gdn_replayssm_enabled() + assert os.environ["GDN_UCACHE_STATE_DTYPE"] == "fp16" + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "bf16") + with pytest.raises(ValueError, match="GDN_UCACHE_STATE_DTYPE='bf16'"): + _gdn_replayssm_config(tmp_path, "align") + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") diff --git a/tests/test_ray_env.py b/tests/test_ray_env.py index 945b2d80b691..5e2d433201e0 100644 --- a/tests/test_ray_env.py +++ b/tests/test_ray_env.py @@ -57,6 +57,16 @@ def test_pythonhashseed_in_result(self): result = get_env_vars_to_copy() assert "PYTHONHASHSEED" in result + def test_gdn_materializer_state_reaches_engine_actor(self, monkeypatch): + monkeypatch.setenv("GDN_UCACHE_STATE_DTYPE", "fp16") + monkeypatch.delenv("VLLM_RAY_EXTRA_ENV_VARS_TO_COPY", raising=False) + monkeypatch.delenv("VLLM_RAY_EXTRA_ENV_VAR_PREFIXES_TO_COPY", raising=False) + names = get_env_vars_to_copy( + destination="EngineCoreActor", exclude_vars=WORKER_SPECIFIC_ENV_VARS + ) + actor_env = {name: os.environ[name] for name in names if name in os.environ} + assert actor_env.get("GDN_UCACHE_STATE_DTYPE") == "fp16" + # --------------------------------------------------------------------------- # User-supplied extensions diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index e078b366c5a2..1cbb1fc1b75d 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -152,12 +152,14 @@ def _create_gdn_builder( num_speculative_tokens: int = 0, full_cuda_graph: bool = False, adaptive_verification: bool = False, + use_replayssm: bool = False, ) -> GDNAttentionMetadataBuilder: """Create a GDNAttentionMetadataBuilder with minimal config.""" vllm_config = create_vllm_config( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE, ) + vllm_config.cache_config.use_replayssm = use_replayssm vllm_config.compilation_config.cudagraph_mode = ( CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE ) @@ -188,6 +190,11 @@ def _build( ) -> GDNAttentionMetadata: """Build GDN attention metadata, optionally with spec-decode kwargs.""" common = create_common_attn_metadata(batch_spec, BLOCK_SIZE, DEVICE) + if builder.use_gdn_replayssm: + common = common.replace( + is_prefilling=torch.zeros(batch_spec.batch_size, dtype=torch.bool), + is_last_prefill=torch.zeros(batch_spec.batch_size, dtype=torch.bool), + ) if block_table is not None: common = common.replace(block_table_tensor=block_table) kwargs: dict = {} @@ -280,6 +287,23 @@ def test_has_initial_state_after_reclassification(): assert meta.has_initial_state[0].item() is True +def test_replayssm_captured_indices_remain_valid_after_ragged_batch(): + """A smaller FULL graph must not read stale indices from a ragged step.""" + builder = _create_gdn_builder( + num_speculative_tokens=3, full_cuda_graph=True, use_replayssm=True + ) + captured = _build(builder, BatchSpec(seq_lens=[80] * 32, query_lens=[4] * 32)) + indices = captured.replayssm_output_indices + assert indices is not None + _build(builder, BatchSpec(seq_lens=[80] * 33, query_lens=[1] + [4] * 32)) + _build(builder, BatchSpec(seq_lens=[80] * 32, query_lens=[4] * 30 + [0] * 2)) + # The graph retains its captured 128-entry view, not the new shorter view. + output = torch.arange(128) + selected = output.index_select(0, indices.long()) + torch.testing.assert_close(selected[:120], output[:120]) + assert selected[120:].eq(0).all() + + def test_full_cudagraph_spec_metadata_uses_request_count(): """FULL cudagraph token padding must not pad request-indexed metadata.""" num_speculative_tokens = 3 diff --git a/tests/v1/attention/test_replayssm_metadata_builder.py b/tests/v1/attention/test_replayssm_metadata_builder.py index 58f00d179a24..0817b3c21824 100644 --- a/tests/v1/attention/test_replayssm_metadata_builder.py +++ b/tests/v1/attention/test_replayssm_metadata_builder.py @@ -18,15 +18,63 @@ create_common_attn_metadata, create_vllm_config, ) -from vllm.config import SpeculativeConfig +from vllm.config import CacheConfig, CompilationConfig, SpeculativeConfig, VllmConfig from vllm.config.compilation import CUDAGraphMode -from vllm.config.mamba import MambaBackendEnum +from vllm.config.mamba import MambaBackendEnum, MambaConfig +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.attention.backends.short_conv_attn import ( + PleShortConvAttentionMetadataBuilder, +) from vllm.v1.kv_cache_interface import MambaSpec BLOCK_SIZE = 16 DEVICE = torch.device("cpu") +@pytest.mark.parametrize( + "mamba_backend", [MambaBackendEnum.TRITON, MambaBackendEnum.FLASHINFER] +) +def test_ple_metadata_stays_canonical_with_replayssm_enabled(mamba_backend): + config = VllmConfig( + cache_config=CacheConfig(use_replayssm=True), + mamba_config=MambaConfig(backend=mamba_backend), + compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL), + ) + builder = PleShortConvAttentionMetadataBuilder( + MambaSpec( + block_size=BLOCK_SIZE, + shapes=((16, 64),), + dtypes=(torch.float32,), + mamba_type=MambaAttentionBackendEnum.SHORT_CONV, + ), + ["ple"], + config, + DEVICE, + ) + assert not builder.use_replayssm + assert not builder.use_flashinfer_replayssm + common = create_common_attn_metadata( + BatchSpec(seq_lens=[17, 18], query_lens=[1, 1]), + BLOCK_SIZE, + DEVICE, + arange_block_indices=True, + ).replace( + is_prefilling=torch.tensor([False, False], dtype=torch.bool), + ) + for metadata in ( + builder.build(0, common), + builder.build_for_cudagraph_capture(common), + ): + torch.testing.assert_close( + metadata.state_indices_tensor, common.block_table_tensor[:, 0] + ) + assert metadata.write_pos_d is None + assert metadata.is_flush_d is None + assert metadata.bc_pre_scratch is None + assert metadata.replayssm_scratch is None + assert metadata.replayssm_state_indices_d is None + + @dataclass class ReplaySSMBuildCase: seq_lens: list[int] diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 498ff7a728ce..d6bc92303b69 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -48,13 +48,18 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata def _make_runner( - req_states: dict[str, tuple[int, int]], decode_query_len: int = 8 + req_states: dict[str, tuple[int, int]], + decode_query_len: int = 8, + use_gdn_replayssm: bool = False, ) -> Any: """Build a runner stub from {req_id: (num_computed_tokens, prefill_len)}.""" prefill_lens = np.array([s[1] for s in req_states.values()], dtype=np.int32) num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32) runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = decode_query_len + runner.vllm_config = SimpleNamespace( + is_gdn_replayssm_enabled=lambda: use_gdn_replayssm + ) runner.adaptive_verification = None runner.pcp_manager = None runner.use_pp = False @@ -154,6 +159,22 @@ def test_one_new_prompt_token_over_context_runs_as_decode(): } +def test_gdn_intermediate_single_token_then_multi_token_prompt_chunk(): + # Token 65 must update canonical state before the final three-token prefill. + runner = _make_runner({"prompt": (0, 68)}, 1, use_gdn_replayssm=True) + for computed, scheduled in [(0, 64), (64, 1), (65, 3)]: + runner.req_states.num_computed_prefill_tokens[0] = computed + scheduler_output = SimpleNamespace( + num_scheduled_tokens={"prompt": scheduled}, + total_num_scheduled_tokens=scheduled, + scheduled_spec_decode_tokens={}, + ) + state, uniform = runner.gather_batch_req_state(scheduler_output, False) + assert state.prefill_runs_as_decode_np.tolist() == [False] + assert not state.decode_graph_eligible + assert uniform is None + + def test_one_token_prompt_tail_with_prior_context_is_uniform_decode(): # Replaying the last prompt token over existing context (full prefix-cache # hit or P/D KV import) computes exactly like a decode. @@ -245,6 +266,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): """ runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = 8 + runner.vllm_config = SimpleNamespace(is_gdn_replayssm_enabled=lambda: False) runner.adaptive_verification = None runner.pcp_manager = None runner.use_pp = False @@ -309,3 +331,12 @@ def test_no_speculator_dispatches_on_query_length_alone(): "these speculators classify a decode batch by query length alone; use " f"get_uniform_decode_token_count instead: {offenders}" ) + + +def test_replayssm_materialization_bound_covers_partial_acceptance(): + from vllm.v1.worker.replayssm_utils import may_cross_mamba_block_boundary + + assert not may_cross_mamba_block_boundary(np.array([16]), np.array([8]), 16) + assert may_cross_mamba_block_boundary(np.array([15]), np.array([1]), 16) + # Four accepted tokens end at the boundary. All eight end after it. + assert may_cross_mamba_block_boundary(np.array([12]), np.array([8]), 16) diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index c80ca7409281..7ce3b729cc9b 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -66,12 +66,14 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non num_tokens_after_padding=1, query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(), query_start_loc=torch.tensor([0, 1], dtype=torch.int32), - num_scheduled_tokens=torch.tensor([1], dtype=torch.int32), + num_scheduled_tokens=np.array([1], dtype=np.int32), max_query_len=None, seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), seq_lens=torch.tensor([1537], dtype=torch.int32), is_prefilling_np=torch.tensor([False]).numpy(), prefill_runs_as_decode_np=None, + num_computed_prefill_tokens_np=np.array([1536]), + prefill_len_np=np.array([1024]), dcp_local_seq_lens=None, positions=positions, prompt_lens=torch.tensor([1024], dtype=torch.int32), @@ -146,6 +148,8 @@ def test_padded_prompt_tail_builds_as_spec_decode( seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), seq_lens=torch.tensor(seq_lens, dtype=torch.int32), is_prefilling_np=np.array(is_prefilling), + num_computed_prefill_tokens_np=np.array([46, 128, 0]), + prefill_len_np=np.array([40, 129, 8]), prefill_runs_as_decode_np=np.array([False, True, False]), dcp_local_seq_lens=None, positions=torch.zeros(12, dtype=torch.int64), @@ -218,6 +222,8 @@ def _input_batch( max_query_len=None, seq_lens_cpu_upper_bound=seq_lens, is_prefilling_np=np.array(is_prefilling), + num_computed_prefill_tokens_np=np.where(is_prefilling, 0, 10), + prefill_len_np=np.where(is_prefilling, 16, 10), prefill_runs_as_decode_np=None, num_draft_tokens_per_req=None if num_draft_tokens_per_req is None @@ -509,10 +515,12 @@ def test_flashinfer_replayssm_prefix_uses_original_accepted_counts() -> None: state = object.__new__(MambaHybridModelState) state._align_mode = True state._use_flashinfer_replayssm = True + state._use_gdn_replayssm = False state.recoverssm = None state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32, device="cuda") state._mamba_state_idx_gpu = torch.zeros(4, dtype=torch.int32, device="cuda") state._is_prefilling_gpu = torch.zeros(4, dtype=torch.bool, device="cuda") + state._replayssm_may_materialize = True state._replayssm_query_start_loc = torch.tensor( [0, 4], dtype=torch.int32, device="cuda" ) diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index 87d3040ec9cb..d7964c676f40 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -556,27 +556,113 @@ def _make_gpu_ctx( ) -def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies(): +@pytest.mark.parametrize( + "mamba_type,wrapped,baseline_type", + [ + (MambaAttentionBackendEnum.MAMBA2, False, None), + (MambaAttentionBackendEnum.GDN_ATTN, False, None), + (MambaAttentionBackendEnum.GDN_ATTN, True, None), + ( + MambaAttentionBackendEnum.GDN_ATTN, + True, + MambaAttentionBackendEnum.SHORT_CONV, + ), + (MambaAttentionBackendEnum.GDN_ATTN, False, MambaAttentionBackendEnum.MAMBA2), + ], +) +def test_replayssm_gpu_copy_layout_preserves_canonical_cpu_copies( + mamba_type, wrapped, baseline_type +): mixer = _TestReplaySSMMixer() + mixer.replayssm_buffer_len = 16 + mixer.replayssm_executed_width = 8 + mixer.A = torch.ones(2) + is_gdn = mamba_type == MambaAttentionBackendEnum.GDN_ATTN + ring_len = 32 if is_gdn else 20 + shapes = ( + (2, 2), + (2, 2, 2), + (2, ring_len, 2), + (1, ring_len, 2) if is_gdn else (2, ring_len), + (2, ring_len) if is_gdn else (1, ring_len, 2), + ) + mixer.kv_cache = tuple(torch.zeros(3, *shape) for shape in shapes) + layer_name = "layers.0.mixer" + forward_context = {layer_name: mixer} spec = MambaSpec( block_size=16, - shapes=mixer.get_state_shape(), + shapes=shapes, dtypes=mixer.get_state_dtype(), + mamba_type=mamba_type, ) kv_config = KVCacheConfig( num_blocks=3, kv_cache_tensors=[], - kv_cache_groups=[KVCacheGroupSpec(["mixer"], spec)], + kv_cache_groups=[ + KVCacheGroupSpec( + [layer_name], + UniformTypeKVCacheSpecs( + block_size=16, kv_cache_specs={layer_name: spec} + ) + if wrapped + else spec, + ) + ], ) + states = list(mixer.kv_cache) + conv_widths = [2, 0, 0, 0, 0] + canonical_states = 2 + if baseline_type is not None: + baseline = _TestReplaySSMMixer() + baseline.use_replayssm = False + baseline.use_flashinfer_replayssm = False + baseline_shapes = shapes[: len(_COPY_FUNCS[baseline_type])] + baseline.kv_cache = tuple(torch.zeros(3, *shape) for shape in baseline_shapes) + forward_context["layers.1.baseline"] = baseline + kv_config.kv_cache_groups.append( + KVCacheGroupSpec( + ["layers.1.baseline"], + MambaSpec( + block_size=16, + shapes=baseline_shapes, + dtypes=(torch.float32,) * len(baseline_shapes), + mamba_type=baseline_type, + tp_replicated=baseline_type == MambaAttentionBackendEnum.SHORT_CONV, + ), + ) + ) + states.extend(baseline.kv_cache) + conv_widths.extend([2] if len(baseline_shapes) == 1 else [2, 0]) + canonical_states += len(baseline_shapes) cfg = _TestConfig(num_layers=1, max_num_reqs=2) device = torch.device("cpu") canonical = _make_copy_bufs(cfg, kv_config, device) - replay = _make_gpu_ctx(cfg, kv_config, device, {"mixer": mixer}) + replay = _make_gpu_ctx(cfg, kv_config, device, forward_context) + initialize_replayssm_ring_trackers(kv_config, forward_context) + block_tables = [ + torch.zeros(cfg.max_num_reqs, 2, dtype=torch.int32) + for _ in kv_config.kv_cache_groups + ] + if baseline_type == MambaAttentionBackendEnum.MAMBA2: + with pytest.raises(ValueError, match="baseline Mamba"): + replay.initialize_from_forward_context( + kv_config, forward_context, _COPY_FUNCS, block_tables + ) + return + replay.initialize_from_forward_context( + kv_config, + forward_context, + _COPY_FUNCS, + block_tables, + ) - assert canonical.src_ptrs.cpu.numel() == cfg.max_num_reqs * 2 - assert replay.num_states == 5 - assert replay.state_base_addrs.numel() == 5 - assert _COPY_FUNCS[MambaAttentionBackendEnum.MAMBA2] == _DEFAULT_COPY_FUNCS + assert canonical.src_ptrs.cpu.numel() == cfg.max_num_reqs * canonical_states + assert replay.num_states == len(states) + assert replay.state_base_addrs.tolist() == [s.data_ptr() for s in states] + assert replay.state_conv_widths.tolist() == conv_widths + assert replay.replayssm is not None + assert replay.replayssm.mamba_type == mamba_type + assert _COPY_FUNCS[mamba_type] == _DEFAULT_COPY_FUNCS def test_replayssm_rejects_incompatible_ring_geometry_before_materialization(): diff --git a/vllm/config/cache.py b/vllm/config/cache.py index ebd4d328793d..4bf0b011f6ec 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -319,16 +319,19 @@ def device_memory_utilization(self, value: float | None) -> None: EAGLE on the Mamba group, and a prefix match unit smaller than the Mamba block size.""" replayssm_buffer_len: int = Field(default=16, gt=0) - """ReplaySSM logical history length B for Mamba2. Triton uses B physical - rows and FlashInfer uses B+T, where T is the target verification length. + """ReplaySSM logical history length B. Mamba2 Triton uses B physical rows + and Mamba2 FlashInfer uses B+T, where T is the verification length. Qwen3.5 + and Qwen4Exp Gated DeltaNet require B=16 and use 32 physical rows. Kimi-K3 speculative decode does not use B. Default 16.""" use_replayssm: bool = False - """Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip - the per-step full-state store, writing the checkpoint back only on flush. - Triton and FlashInfer support the 'none' and 'align' Mamba cache modes. - Mamba2 speculative decoding requires FlashInfer. Pipeline parallelism is not - supported. Flushes are most efficient when mamba_block_size is a - multiple of replayssm_buffer_len, but this is not required.""" + """Use a supported ReplaySSM decode kernel: cache recent recurrent inputs + and skip the per-step full-state store, writing the checkpoint only on flush. + Mamba2 supports Triton and FlashInfer in 'none' and 'align' cache modes. + Mamba2 speculative decoding, Qwen3.5 Gated DeltaNet, and Qwen4Exp Gated + DeltaNet require FlashInfer. Qwen4Exp ReplaySSM does not support ROCm. + Pipeline parallelism is not supported. Flushes are most efficient when + mamba_block_size is a multiple of replayssm_buffer_len, but this is not + required.""" use_kda_recoverssm: bool = field(default=False, init=False) """Whether Kimi-K3 KDA uses RecoverSSM speculative decode.""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 118a289af35e..e7390cc88d02 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -132,6 +132,30 @@ } ) +# SupportsReplaySSM advertises support for some ReplaySSM implementation. This +# allowlist defines the current FlashInfer GDN integration scope. Sharing a +# layer or inheriting the marker does not opt another model into this integration. +_GDN_REPLAYSSM_ARCHITECTURES = frozenset( + { + "Qwen3_5ForCausalLM", + "Qwen3_5MoeForCausalLM", + "Qwen3_5ForConditionalGeneration", + "Qwen3_5MoeForConditionalGeneration", + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + } +) + + +def _uses_gdn_replayssm( + cache_config: CacheConfig, model_config: "ModelConfig | None" +) -> bool: + return bool( + cache_config.use_replayssm + and model_config is not None + and model_config.architecture in _GDN_REPLAYSSM_ARCHITECTURES + ) + @lru_cache def default_breakable_cudagraph_architectures() -> frozenset[str]: @@ -577,9 +601,10 @@ def max_in_flight_tokens(self) -> int: @property def use_flashinfer_replayssm(self) -> bool: - return ( - self.cache_config.use_replayssm - and self.mamba_config.backend == MambaBackendEnum.FLASHINFER + """Whether a FlashInfer ReplaySSM kernel runs Mamba2 or GDN decode.""" + return self.cache_config.use_replayssm and ( + _uses_gdn_replayssm(self.cache_config, self.model_config) + or self.mamba_config.backend == MambaBackendEnum.FLASHINFER ) @property @@ -3582,11 +3607,104 @@ def validate_mamba_block_size(self) -> "VllmConfig": ) return self + @model_validator(mode="after") + def validate_gdn_replayssm(self) -> "VllmConfig": + if not _uses_gdn_replayssm(self.cache_config, self.model_config): + return self + if ( + self.model_config.architecture.startswith("Qwen4Exp") + and current_platform.is_rocm() + ): + raise ValueError( + "FlashInfer GDN ReplaySSM does not support Qwen4Exp on ROCm" + ) + cache_config = self.cache_config + if not 0 <= self.num_speculative_tokens <= 7: + raise ValueError( + "FlashInfer GDN ReplaySSM supports at most 7 speculative tokens" + ) + if ( + self.speculative_config is not None + and self.speculative_config.enable_adaptive_verification + ): + raise ValueError( + "FlashInfer GDN ReplaySSM does not support adaptive verification; " + "set enable_adaptive_verification=False" + ) + if cache_config.replayssm_buffer_len != 16: + raise ValueError( + "FlashInfer GDN ReplaySSM requires --replayssm-buffer-len=16" + ) + if cache_config.mamba_cache_mode not in ("none", "align"): + raise ValueError( + "FlashInfer GDN ReplaySSM supports only the none and align " + "Mamba cache modes" + ) + if ( + cache_config.enable_prefix_caching + and cache_config.mamba_cache_mode != "align" + ): + raise ValueError( + "FlashInfer GDN ReplaySSM prefix caching requires " + "--mamba-cache-mode=align" + ) + if self.model_config.dtype != torch.bfloat16: + raise ValueError("FlashInfer GDN ReplaySSM requires bfloat16 model dtype") + hf_config = self.model_config.hf_text_config + if ( + getattr(hf_config, "linear_key_head_dim", None) != 128 + or getattr(hf_config, "linear_value_head_dim", None) != 128 + ): + raise ValueError("FlashInfer GDN ReplaySSM requires GDN K=V=128") + if cache_config.mamba_cache_dtype not in ("auto", "bfloat16"): + raise ValueError( + "FlashInfer GDN ReplaySSM requires a bfloat16 convolution cache" + ) + if cache_config.mamba_ssm_cache_dtype != "float16": + raise ValueError( + "FlashInfer GDN ReplaySSM requires --mamba-ssm-cache-dtype float16" + ) + if self.mamba_config.enable_stochastic_rounding: + raise ValueError( + "FlashInfer GDN ReplaySSM does not support " + "--enable-mamba-cache-stochastic-rounding" + ) + # Replay rings and cursors are not part of the pipeline, microbatch, or + # KV-connector state handoff. + if self.parallel_config.pipeline_parallel_size > 1: + raise ValueError( + "FlashInfer GDN ReplaySSM currently requires pipeline_parallel_size=1" + ) + if self.parallel_config.use_ubatching: + raise ValueError( + "FlashInfer GDN ReplaySSM does not support overlapping " + "microbatch execution" + ) + if ( + self.kv_transfer_config is not None + and self.kv_transfer_config.is_kv_transfer_instance + ): + raise ValueError( + "--use-replayssm is incompatible with KV connectors " + "(P/D disaggregation, KV cache offload)" + ) + if cache_config.mamba_cache_mode == "align": + from vllm.model_executor.layers.mamba.mamba_utils import ( + configure_flashinfer_gdn_env, + ) + + configure_flashinfer_gdn_env() + cache_config.use_kda_recoverssm = False + return self + @model_validator(mode="after") def validate_mamba_cached_kernel(self) -> "VllmConfig": if not self.cache_config.use_replayssm: self.cache_config.use_kda_recoverssm = False return self + # validate_gdn_replayssm owns the GDN checks. + if _uses_gdn_replayssm(self.cache_config, self.model_config): + return self kda_architectures = ( "KimiLinearForCausalLM", @@ -3677,6 +3795,10 @@ def validate_mamba_cached_kernel(self) -> "VllmConfig": ) return self + def is_gdn_replayssm_enabled(self) -> bool: + """Whether --use-replayssm selects FlashInfer GDN ReplaySSM.""" + return _uses_gdn_replayssm(self.cache_config, self.model_config) + _current_vllm_config: VllmConfig | None = None _current_prefix: str | None = None diff --git a/vllm/model_executor/layers/mamba/gdn/base.py b/vllm/model_executor/layers/mamba/gdn/base.py index 249d56685ee7..43ba7c0572a6 100644 --- a/vllm/model_executor/layers/mamba/gdn/base.py +++ b/vllm/model_executor/layers/mamba/gdn/base.py @@ -14,6 +14,8 @@ from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, + gdn_replayssm_executed_width, + mamba_state_alignment, ) from vllm.model_executor.models.utils import extract_layer_index from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum @@ -45,14 +47,48 @@ def __init__( if self.speculative_config else 0 ) + self.use_replayssm = vllm_config.is_gdn_replayssm_enabled() + # GDN ReplaySSM always runs on FlashInfer; MambaBase consumers read this. + self.use_flashinfer_replayssm = self.use_replayssm + self.state_alignment = mamba_state_alignment(vllm_config) + self.replayssm_buffer_len = ( + self.cache_config.replayssm_buffer_len if self.use_replayssm else None + ) + self.replayssm_executed_width: int | None = None + if self.use_replayssm: + self.replayssm_executed_width = gdn_replayssm_executed_width(self.num_spec) + self.register_buffer( + "_replayssm_offsets", + torch.arange(self.replayssm_executed_width, dtype=torch.int32), + persistent=False, + ) + self._replayssm_ring_start = torch.empty(0, dtype=torch.int32) + self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32) @property def mamba_type(self) -> MambaAttentionBackendEnum: return MambaAttentionBackendEnum.GDN_ATTN + @property + def replayssm_rings(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Bound `(u, k, G)` rings, stored after the conv and SSM states.""" + u_cache, k_cache, g_cache = self.kv_cache[2:5] + return u_cache, k_cache, g_cache + def get_state_dtype(self) -> tuple[torch.dtype, ...]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - self.model_config.dtype, - self.cache_config.mamba_cache_dtype, - self.cache_config.mamba_ssm_cache_dtype, + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + self.model_config.dtype, + self.cache_config.mamba_cache_dtype, + self.cache_config.mamba_ssm_cache_dtype, + ) ) + + if self.use_replayssm: + dtypes = ( + *dtypes, + *MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + self.model_config.dtype + ), + ) + return dtypes diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 77c284449d5a..c8f4d5237e6e 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -44,6 +44,11 @@ MambaPrefillCheckpointExporter, ) from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_kernel, + load_gdn_replayssm_materializer, + run_gdn_replayssm, +) from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, @@ -683,8 +688,8 @@ def forward_hip( class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): def get_state_shape( self, - ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - return MambaStateShapeCalculator.gated_delta_net_state_shape( + ) -> tuple[tuple[int, ...], ...]: + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( self.tp_size, self.num_k_heads, self.num_v_heads, @@ -694,6 +699,19 @@ def get_state_shape( self.num_spec, ) + if self.use_replayssm: + shapes = ( + *shapes, + *MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + self.tp_size, + self.num_k_heads, + self.num_v_heads, + self.head_k_dim, + self.head_v_dim, + ), + ) + return shapes + def __init__( self, config: Qwen3NextConfig, @@ -866,7 +884,14 @@ def __init__( # returns the out_proj input: `out`, or a pre-quantized activation. self._fused_gdn_gated_rms_norm = fused_gdn_gated_rms_norm self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower() - if self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): + if self.use_replayssm: + reason = self._flashinfer_replayssm_unsupported_reason() + if reason is not None: + raise ValueError(f"--use-replayssm is not supported: {reason}") + self._replayssm_kernel = load_gdn_replayssm_kernel() + if self.cache_config.mamba_cache_mode == "align": + load_gdn_replayssm_materializer() + elif self.gdn_decode_kernel == "cuda" and current_platform.is_cuda_alike(): reason = self._fused_gdn_decode_unsupported_reason(vllm_config) if reason is not None: if "VLLM_GDN_DECODE_KERNEL" in os.environ: @@ -895,6 +920,19 @@ def __init__( raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + def _flashinfer_replayssm_unsupported_reason(self) -> str | None: + # validate_gdn_replayssm checks dtypes and head sizes from the config. + if not current_platform.is_device_capability_family(100): + return "FlashInfer's UMMA kernel requires an SM100-class GPU" + if self.gqa_interleaved_layout: + return "the interleaved GQA layout is not supported" + if self.num_k_heads % self.tp_size or self.num_v_heads % self.tp_size: + return "tensor-parallel head counts must divide evenly" + heads_per_key, remainder = divmod(self.num_v_heads, self.num_k_heads) + if remainder or heads_per_key > 4: + return "FlashInfer's UMMA kernel needs 1 to 4 value heads per key head" + return None + def get_prefill_checkpoint_alignment(self) -> int | None: return self._checkpoint_alignment @@ -941,7 +979,7 @@ def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]: def _fused_gdn_decode_unsupported_reason( self, vllm_config: VllmConfig ) -> str | None: - conv_state_dtype, recurrent_state_dtype = self.get_state_dtype() + conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()[:2] if ( self.gqa_interleaved_layout or self.head_k_dim != 128 @@ -1332,6 +1370,7 @@ def forward_cuda( use_fused_gdn_spec_decode = ( self.enable_fused_gdn_spec_decode + and not self.use_replayssm and hidden_states.dtype == torch.bfloat16 and self.norm.weight.dtype in (torch.bfloat16, torch.float32) ) @@ -1550,7 +1589,7 @@ def _warmup_prefill_kernels(self, qkv_or_qkvz: torch.Tensor, v_dim: int) -> None dtype = qkv_or_qkvz.dtype num_k_heads = self.num_k_heads // self.tp_size num_v_heads = self.num_v_heads // self.tp_size - _, state_dtype = self.get_state_dtype() + _, state_dtype = self.get_state_dtype()[:2] # All kernels use BT = chunk_size, so a single pass with T = chunk_size # is sufficient to populate every autotuner cache. Mirror the real @@ -1795,6 +1834,7 @@ def _forward_core( if ( self.enable_packed_recurrent_decode + and not self.use_replayssm and attn_metadata.spec_sequence_masks is None and attn_metadata.num_prefills == 0 and attn_metadata.num_decodes > 0 @@ -1866,7 +1906,9 @@ def _forward_core( ], num_accepted_tokens=num_accepted_tokens, query_start_loc=spec_query_start_loc, - max_query_len=spec_state_indices_tensor.size(-1), + max_query_len=( + self.replayssm_executed_width or spec_state_indices_tensor.size(-1) + ), validate_data=False, ) @@ -1904,7 +1946,11 @@ def _forward_core( else: mixed_qkv_non_spec = None - query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) + if self.use_replayssm and spec_sequence_masks is not None: + # run_gdn_replayssm splits the packed QKV itself. + query_spec, key_spec, value_spec = None, None, None + else: + query_spec, key_spec, value_spec = self.rearrange_mixed_qkv(mixed_qkv_spec) # Split mixed non-spec-decode+prefill to process independently split_non_spec = ( @@ -1967,7 +2013,38 @@ def _forward_core( # 2. Recurrent attention # 2.1: Process the multi-query part - if spec_sequence_masks is not None: + if self.use_replayssm and spec_sequence_masks is not None: + assert mixed_qkv_spec is not None + assert a_spec is not None and b_spec is not None + assert spec_state_indices_tensor is not None + assert spec_query_start_loc is not None + assert attn_metadata.replayssm_output_indices is not None + assert self.replayssm_executed_width is not None + num_replay_rows = attn_metadata.num_spec_decodes + core_attn_out_spec = run_gdn_replayssm( + kernel=self._replayssm_kernel, + executed_width=self.replayssm_executed_width, + mixed_qkv=mixed_qkv_spec, + a=a_spec, + b=b_spec, + A_log=self.A_log, + dt_bias=self.dt_bias, + checkpoint_state=ssm_state, + replayssm_cache=self.replayssm_rings, + ring_start=self._replayssm_ring_start, + num_committed=self._replayssm_prev_num_accepted, + query_start_loc=spec_query_start_loc[: num_replay_rows + 1], + state_indices=spec_state_indices_tensor[:num_replay_rows, 0], + output_indices=attn_metadata.replayssm_output_indices, + uniform_query_length=attn_metadata.uniform_spec_sequence_length, + num_k_heads=self.num_k_heads // self.tp_size, + num_v_heads=self.num_v_heads // self.tp_size, + head_k_dim=self.head_k_dim, + head_v_dim=self.head_v_dim, + offsets=self._replayssm_offsets, + ).unsqueeze(0) + last_recurrent_state = None + elif spec_sequence_masks is not None: core_attn_out_spec, last_recurrent_state = ( fused_sigmoid_gating_delta_rule_update( A_log=self.A_log, @@ -2110,7 +2187,15 @@ def _forward_core( 0, non_spec_token_indx, core_attn_out_non_spec.squeeze(0) ) elif spec_sequence_masks is not None: - core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0) + if self.use_replayssm: + # FULL graphs pad tokens after the real replay rows; scatter + # only the real-token outputs. + assert spec_token_indx is not None + core_attn_out.index_copy_( + 0, spec_token_indx.long(), core_attn_out_spec.squeeze(0) + ) + else: + core_attn_out[:num_actual_tokens] = core_attn_out_spec.squeeze(0) elif not prefill_in_place: core_attn_out[:num_actual_tokens] = core_attn_out_non_spec.squeeze(0) @@ -2427,7 +2512,8 @@ def _can_use_fused_gdn_mtp_decode( ) -> bool: state_indices = attn_metadata.spec_state_indices_tensor return ( - attn_metadata.spec_sequence_masks is not None + not self.use_replayssm + and attn_metadata.spec_sequence_masks is not None and attn_metadata.num_decodes == 0 and attn_metadata.num_spec_decodes > 0 and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES diff --git a/vllm/model_executor/layers/mamba/gdn/replayssm.py b/vllm/model_executor/layers/mamba/gdn/replayssm.py new file mode 100644 index 000000000000..ff2c7d325b98 --- /dev/null +++ b/vllm/model_executor/layers/mamba/gdn/replayssm.py @@ -0,0 +1,213 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapter for FlashInfer's FP16-state GDN ReplaySSM decode kernel.""" + +from collections.abc import Callable +from functools import cache + +import torch + +from vllm.model_executor.layers.mamba.mamba_utils import GDN_REPLAYSSM_LOGICAL_WINDOW +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + +# CuTe loads every kernel input with 16-byte vector accesses. +_CUTE_ALIGNMENT_BYTES = 16 + + +@cache +def load_gdn_replayssm_kernel() -> Callable[..., None]: + """Import FlashInfer's SM100 UMMA ReplaySSM decode kernel.""" + try: + from flashinfer.gdn_kernels.gdn_replay_mtp_umma import ( + gated_delta_rule_mtp_ucache_flush_umma, + ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM requires " + "gated_delta_rule_mtp_ucache_flush_umma (FlashInfer >= 0.7.3)" + ) from e + return gated_delta_rule_mtp_ucache_flush_umma + + +@cache +def load_gdn_replayssm_materializer() -> Callable[..., None]: + """Import FlashInfer's prefix materializer and check its FP16 state dtype.""" + try: + from flashinfer.gdn_kernels.gdn_prefix_materialize import ( + gdn_prefix_materialize, + state_ty_torch, + ) + except ImportError as e: + raise ImportError( + "FlashInfer GDN ReplaySSM prefix caching requires " + "flashinfer.gdn_kernels.gdn_prefix_materialize (FlashInfer >= 0.7.3)" + ) from e + if state_ty_torch() != torch.float16: + raise ValueError( + f"FlashInfer GDN prefix materialization uses {state_ty_torch()} " + "state because FlashInfer was imported before vLLM configured it. " + "Set GDN_UCACHE_STATE_DTYPE=fp16 before starting the process." + ) + return gdn_prefix_materialize + + +def pack_replayssm_rows( + values: torch.Tensor, + query_start_loc: torch.Tensor, + executed_width: int, + offsets: torch.Tensor | None = None, + padding_value: float = 0.0, + uniform: bool = False, + allow_strided_rows: bool = False, +) -> torch.Tensor: + """Pack variable-length token rows into `[num_rows, executed_width, ...]`. + + Row `i` holds the tokens in `query_start_loc[i]:query_start_loc[i + 1]` + followed by `padding_value`. With `uniform`, every row has exactly + `executed_width` tokens, so the result is a view of `values` when it + is contiguous and aligned for CuTe, and a contiguous copy otherwise. + `allow_strided_rows` also preserves aligned, non-overlapping 2D rows. + """ + num_rows = query_start_loc.numel() - 1 + if uniform and values.shape[0] == num_rows * executed_width: + aligned_rows = ( + allow_strided_rows + and values.ndim == 2 + and values.stride(1) == 1 + and values.stride(0) >= values.shape[1] + and values.stride(0) * values.element_size() % _CUTE_ALIGNMENT_BYTES == 0 + ) + if values.data_ptr() % _CUTE_ALIGNMENT_BYTES or not ( + values.is_contiguous() or aligned_rows + ): + values = values.clone(memory_format=torch.contiguous_format) + return values.view(num_rows, executed_width, *values.shape[1:]) + if num_rows == 0: + return values.new_empty((0, executed_width, *values.shape[1:])) + if values.shape[0] == 0: + raise ValueError("cannot pack non-empty ReplaySSM rows from an empty tensor") + if offsets is None: + offsets = torch.arange( + executed_width, dtype=query_start_loc.dtype, device=query_start_loc.device + ) + else: + offsets = offsets[:executed_width] + row_offsets = offsets.unsqueeze(0).expand(num_rows, -1) + query_lens = torch.diff(query_start_loc).unsqueeze(1) + valid = row_offsets < query_lens + source_indices = query_start_loc[:-1].unsqueeze(1) + row_offsets + source_indices = source_indices.clamp(max=values.shape[0] - 1) + packed = values.index_select(0, source_indices.reshape(-1).long()).view( + num_rows, executed_width, *values.shape[1:] + ) + mask = valid.reshape(num_rows, executed_width, *([1] * (values.dim() - 1))) + return packed.masked_fill(~mask, padding_value) + + +def run_gdn_replayssm( + *, + kernel: Callable[..., None], + executed_width: int, + mixed_qkv: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + checkpoint_state: torch.Tensor, + replayssm_cache: tuple[torch.Tensor, torch.Tensor, torch.Tensor], + ring_start: torch.Tensor, + num_committed: torch.Tensor, + query_start_loc: torch.Tensor, + state_indices: torch.Tensor, + output_indices: torch.Tensor, + num_k_heads: int, + num_v_heads: int, + head_k_dim: int, + head_v_dim: int, + offsets: torch.Tensor | None = None, + uniform_query_length: int | None = None, +) -> torch.Tensor: + """Run one FlashInfer decode step over the replay rows. + + Returns the outputs of real tokens in `output_indices` order. The kernel + reads the committed cursors but does not publish them; vLLM commits the + accepted tokens once per model step, after every layer runs. + """ + uniform = uniform_query_length == executed_width + packed_qkv = pack_replayssm_rows( + mixed_qkv, + query_start_loc, + executed_width, + offsets, + uniform=uniform, + # UMMA makes contiguous Q/K/V copies after splitting the packed view. + allow_strided_rows=True, + ) + # Padded token lanes must not change the state: a=-inf makes the decay + # exp(-softplus(a + dt_bias) * exp(A_log)) one, and zero q/k/v/b make the + # update zero. a and b are packed separately so each gets an aligned + # allocation. + packed_a = pack_replayssm_rows( + a, + query_start_loc, + executed_width, + offsets, + padding_value=-torch.inf, + uniform=uniform, + ) + packed_b = pack_replayssm_rows( + b, query_start_loc, executed_width, offsets, uniform=uniform + ) + local_k_dim = num_k_heads * head_k_dim + local_v_dim = num_v_heads * head_v_dim + q_flat, k_flat, v_flat = packed_qkv.split( + (local_k_dim, local_k_dim, local_v_dim), dim=-1 + ) + num_rows = packed_qkv.size(0) + q = q_flat.view(num_rows, executed_width, num_k_heads, head_k_dim) + k = k_flat.view_as(q) + v = v_flat.view(num_rows, executed_width, num_v_heads, head_v_dim) + # vLLM pads with block zero; FlashInfer skips only negative state indices. + # Gather cursors through the padding slot, then pass -1 for padding rows. + valid_slots = state_indices > NULL_BLOCK_ID + safe_slots = torch.where(valid_slots, state_indices, NULL_BLOCK_ID).long() + fi_state_indices = torch.where(valid_slots, state_indices, -1) + hist_len = num_committed.index_select(0, safe_slots).masked_fill(~valid_slots, 0) + cache_base = ring_start.index_select(0, safe_slots).masked_fill(~valid_slots, 0) + u_cache, k_cache, g_cache = replayssm_cache + # Zero, not empty: padded token rows keep flowing through the model. + output = torch.zeros( + num_rows, + executed_width, + num_v_heads, + head_v_dim, + dtype=mixed_qkv.dtype, + device=mixed_qkv.device, + ) + kernel( + # Detached views, not the Parameters: CuTe rejects tensors that require + # grad, and FlashInfer caches its dtype cast by tensor object. A new + # view per call picks up in-place weight reloads. + A_log=A_log.detach(), + a=packed_a, + dt_bias=dt_bias.detach(), + q=q, + k=k, + v=v, + b=packed_b, + initial_state_source=checkpoint_state, + initial_state_indices=fi_state_indices, + use_qk_l2norm_in_kernel=True, + scale=head_k_dim**-0.5, + output=output, + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + hist_len=hist_len, + cache_base=cache_base, + # Fold history into the checkpoint once history plus this launch's + # tokens would exceed the logical window. + flush_min=GDN_REPLAYSSM_LOGICAL_WINDOW + 1 - executed_width, + restart_hist_on_flush=False, + ) + return output.flatten(0, 1).index_select(0, output_indices.long()) diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 956ccf21f14e..5d3c7eaf8134 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +import os from collections.abc import Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Literal, TypeAlias @@ -28,6 +29,49 @@ ConvStateLayoutType = Literal["SD", "DS"] +GDN_REPLAYSSM_LOGICAL_WINDOW = 16 +"""Committed tokens per request kept in the GDN ReplaySSM rings. Older history +is folded into the checkpoint state.""" +GDN_REPLAYSSM_RING_SLOTS = 32 +"""Physical u/k/G ring slots per request. FlashInfer's UMMA kernel appends up +to eight tokens after the 16-token window and takes a 32-slot ring.""" + + +def gdn_replayssm_executed_width(num_speculative_tokens: int) -> int: + """Return the tokens per request that one GDN ReplaySSM launch computes. + + Shorter rows are padded with no-op tokens up to this width. + """ + if num_speculative_tokens == 0: + return 1 + if 1 <= num_speculative_tokens <= 3: + return 4 + if 4 <= num_speculative_tokens <= 7: + return 8 + raise ValueError( + "FlashInfer GDN ReplaySSM supports 0 through 7 speculative tokens; got " + f"{num_speculative_tokens}" + ) + + +def configure_flashinfer_gdn_env() -> None: + """Select FP16 state in FlashInfer's GDN prefix materializer. + + FlashInfer reads `GDN_UCACHE_STATE_DTYPE` once, when `flashinfer` is + first imported, so it must be set before vLLM starts its workers. + + Raises: + ValueError: The environment already selects another state dtype. + + """ + state = os.environ.setdefault("GDN_UCACHE_STATE_DTYPE", "fp16") + if state.strip().lower() not in ("fp16", "float16", "half"): + raise ValueError( + f"GDN_UCACHE_STATE_DTYPE={state!r} conflicts with the FP16 recurrent " + "state cache that GDN ReplaySSM uses; unset it" + ) + + def mamba_state_alignment(vllm_config: "VllmConfig") -> int: # FlashInfer loads state and ring caches with 16-byte cp.async instructions. return 16 if vllm_config.use_flashinfer_replayssm else 1 @@ -137,6 +181,15 @@ def gated_delta_net_state_dtype( model_dtype, mamba_cache_dtype, mamba_ssm_cache_dtype ) + @classmethod + def gated_delta_net_replayssm_ring_dtypes( + cls, + model_dtype: ModelDType | torch.dtype, + ) -> tuple[torch.dtype, torch.dtype, torch.dtype]: + """Return FlashInfer GDN `(u, k, G)` ring dtypes.""" + activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype) + return (activation_dtype, activation_dtype, torch.float32) + @classmethod def kda_state_dtype( cls, @@ -300,6 +353,25 @@ def gated_delta_net_state_shape( ) return conv_state_shape, temporal_state_shape + @classmethod + def gated_delta_net_replayssm_ring_shapes( + cls, + tp_world_size: int, + num_k_heads: int, + num_v_heads: int, + head_k_dim: int, + head_v_dim: int, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + """Return FlashInfer GDN `(u, k, G)` ring shapes.""" + local_k_heads = divide(num_k_heads, tp_world_size) + local_v_heads = divide(num_v_heads, tp_world_size) + slots = GDN_REPLAYSSM_RING_SLOTS + return ( + (local_v_heads, slots, head_v_dim), + (local_k_heads, slots, head_k_dim), + (local_v_heads, slots), + ) + @classmethod def kda_state_shape( cls, diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 68dd870ccbdb..86b474b56373 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -85,7 +85,14 @@ """ MambaStateShapes: TypeAlias = ( - tuple[tuple[int, int]] + tuple[ + tuple[int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int, int], + tuple[int, int], + ] + | tuple[tuple[int, int]] | tuple[tuple[int, int, int]] | tuple[tuple[int, int], tuple[int, int]] | tuple[tuple[int, int], tuple[int, int, int]] diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 4a6e2ec4361a..ceb37facb0a0 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -66,12 +66,14 @@ from .interfaces import ( HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsEagle3, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from .qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP @@ -301,6 +303,7 @@ class Qwen3_5ForCausalLMBase( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, ): packed_modules_mapping = { "qkv_proj": [ @@ -386,17 +389,25 @@ def forward( def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) ) + if vllm_config.is_gdn_replayssm_enabled(): + dtypes += MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + vllm_config.model_config.dtype + ) + return dtypes + @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: parallel_config = vllm_config.parallel_config hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size @@ -405,7 +416,7 @@ def get_mamba_state_shape_from_config( if vllm_config.speculative_config else 0 ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, @@ -415,6 +426,16 @@ def get_mamba_state_shape_from_config( num_spec, ) + if vllm_config.is_gdn_replayssm_enabled(): + shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + ) + return shapes + @classmethod def get_mamba_state_copy_func( cls, @@ -468,7 +489,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): info=Qwen3_5ProcessingInfo, dummy_inputs=Qwen3VLDummyInputsBuilder, ) -class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid): +class Qwen3_5ForConditionalGeneration( + Qwen3VLForConditionalGeneration, IsHybrid, SupportsReplaySSM +): supports_multimodal_pruning = True hf_to_vllm_mapper = ( @@ -607,34 +630,16 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, - ) + ) -> tuple[torch.dtype, ...]: + # The language model owns the recurrent layers. Reuse its cache schema, + # including ReplaySSM rings, for this multimodal wrapper. + return Qwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, - ) + ) -> MambaStateShapes: + return Qwen3_5ForCausalLMBase.get_mamba_state_shape_from_config(vllm_config) @classmethod def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index ec9c162ee3f7..af6985fc94d6 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -40,6 +40,7 @@ from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup from vllm.model_executor.warmup.qwen_vl_triton_warmup import qwen_vl_triton_warmup from vllm.model_executor.warmup.replayssm_warmup import ( + gdn_replayssm_warmup, replayssm_autotune_warmup, replayssm_lifecycle_warmup, ) @@ -364,6 +365,8 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False): elif has_flashinfer() and current_platform.has_device_capability(90): flashinfer_autotune(worker.model_runner) + # GDN ReplaySSM compiles CuTeDSL artifacts without FlashInfer autotuning. + gdn_replayssm_warmup(worker.model_runner) _warmup_flashinfer_attention(worker) diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index e7963c7181c7..17d35c4cdf42 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -8,6 +8,7 @@ import torch from vllm.logger import init_logger +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_executed_width from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( flashinfer_replayssm_autotune_supported, ) @@ -23,7 +24,7 @@ def _replayssm_autotune_kwargs( max_token_prefill_kwargs: dict[str, Any], ) -> tuple[int, dict[str, Any]] | None: config = runner.vllm_config - if not config.use_flashinfer_replayssm: + if not config.use_flashinfer_replayssm or config.is_gdn_replayssm_enabled(): return None if not flashinfer_replayssm_autotune_supported(): logger.info_once( @@ -64,14 +65,14 @@ def _temporary_replayssm_autotune_state( *, prepare_dummy_block_tables: bool = True, ) -> Iterator[None]: - from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 + """Run FlashInfer ReplaySSM dummy steps, then restore the touched state.""" + from vllm.model_executor.layers.mamba.abstract import MambaBase reset_tensors: list[torch.Tensor] = [] reset_trackers: dict[int, torch.Tensor] = {} for module in runner.get_model().modules(): - if not isinstance(module, MambaMixer2) or not module.use_replayssm: + if not isinstance(module, MambaBase) or not module.use_flashinfer_replayssm: continue - assert module.replayssm_buffer_len is not None ring_start = module._replayssm_ring_start prev_num_accepted = module._replayssm_prev_num_accepted for tensor in module.kv_cache: @@ -109,16 +110,13 @@ def replayssm_lifecycle_warmup(runner: "GPUModelRunner") -> None: return from vllm.model_executor.layers.mamba.abstract import MambaBase - from vllm.v1.kv_cache_interface import MambaSpec + from vllm.v1.worker.mamba_utils import _get_mamba_spec_for_layer from vllm.v1.worker.replayssm_utils import ReplaySSMModelContext grouped = [] for group_id, group in enumerate(runner.kv_cache_config.kv_cache_groups): - spec = group.kv_cache_spec - if not isinstance(spec, MambaSpec): - continue - mixers = [ - layer + names = [ + name for name in group.layer_names if isinstance( layer := config.compilation_config.static_forward_context[name], @@ -126,8 +124,12 @@ def replayssm_lifecycle_warmup(runner: "GPUModelRunner") -> None: ) and layer.use_flashinfer_replayssm ] - if not mixers: + if not names: continue + spec = _get_mamba_spec_for_layer(group, names[0]) + mixers = [ + config.compilation_config.static_forward_context[name] for name in names + ] table = runner.block_tables.input_block_tables[group_id] # Match the runtime stride without changing its block-table contents. scratch_table = torch.empty_strided( @@ -205,3 +207,97 @@ def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None: ensure_kv_cache_blocks(runner, 1 + max_num_reqs) with _temporary_replayssm_autotune_state(runner, max_num_reqs): runner._dummy_run(**decode_kwargs) + + +def _warm_gdn_replayssm_materializer( + runner: "GPUModelRunner", max_num_reqs: int +) -> None: + """Compile the GDN prefix materializer for each current cache layout. + + FlashInfer specializes on state and ring shapes and strides. A later + pool-capacity change can require another compilation. + """ + from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention + from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_materializer, + ) + + materialize = load_gdn_replayssm_materializer() + warmed: set[tuple[Any, ...]] = set() + for module in runner.get_model().modules(): + if not (isinstance(module, GatedDeltaNetAttention) and module.use_replayssm): + continue + state = module.kv_cache[1] + u_cache, k_cache, g_cache = module.replayssm_rings + signature = tuple( + (t.device, t.dtype, tuple(t.shape), tuple(t.stride())) + for t in (state, u_cache, k_cache, g_cache) + ) + if signature in warmed: + continue + warmed.add(signature) + # Every request index is inactive, so the kernel touches no state. + inactive = torch.full( + (max_num_reqs,), -1, dtype=torch.int32, device=state.device + ) + materialize( + state=state, + src_slots=inactive, + dst_slots=inactive, + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + cache_base=torch.zeros_like(inactive), + count=inactive, + active_request_indices=inactive, + num_active=torch.zeros(1, dtype=torch.int32, device=state.device), + ) + + +def gdn_replayssm_warmup(runner: "GPUModelRunner") -> None: + """Warm captured GDN decode sizes and current materializer layouts.""" + config = runner.vllm_config + if not config.is_gdn_replayssm_enabled(): + return + from vllm.v1.worker.extensible_kv_cache import ( + ensure_kv_cache_blocks, + num_committable_kv_blocks, + ) + + query_len = config.num_speculative_tokens + 1 + assert query_len <= gdn_replayssm_executed_width(config.num_speculative_tokens) + max_num_reqs = min( + runner.scheduler_config.max_num_seqs, + num_committable_kv_blocks(runner) - 1, + ) + if max_num_reqs <= 0: + raise ValueError( + "FlashInfer GDN ReplaySSM warmup requires a non-padding state slot" + ) + max_decode_reqs = min(max_num_reqs, runner.max_num_tokens // query_len) + manager = runner.cudagraph_manager + assert manager is not None + capture_reqs = { + desc.num_reqs + for descs in manager._capture_descs.values() + for desc in descs + if desc.uniform_token_count == query_len + and desc.num_reqs is not None + and 0 < desc.num_reqs <= max_decode_reqs + } + if not capture_reqs and max_decode_reqs > 0: + capture_reqs.add(max_decode_reqs) + + decode_kwargs: dict[str, Any] = { + "uniform_decode": True, + "skip_eplb": True, + "is_profile": True, + "randomize_inputs": True, + "valid_dummy_state_slots": True, + } + ensure_kv_cache_blocks(runner, 1 + max_decode_reqs) + with _temporary_replayssm_autotune_state(runner, max_decode_reqs): + for num_reqs in sorted(capture_reqs): + runner._dummy_run(num_tokens=num_reqs * query_len, **decode_kwargs) + if config.cache_config.mamba_cache_mode == "align": + _warm_gdn_replayssm_materializer(runner, max_num_reqs) diff --git a/vllm/models/qwen3_5/nvidia/model.py b/vllm/models/qwen3_5/nvidia/model.py index 26eff345dcf6..727ceef5bf26 100644 --- a/vllm/models/qwen3_5/nvidia/model.py +++ b/vllm/models/qwen3_5/nvidia/model.py @@ -62,8 +62,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, MambaStateCopyFuncCalculator, - MambaStateDtypeCalculator, - MambaStateShapeCalculator, ) from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( nvfp4_weight_padding_bytes, @@ -81,15 +79,20 @@ EagleModelMixin, HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsEagle3, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP +from vllm.model_executor.models.qwen3_5 import ( + Qwen3_5ForCausalLMBase as CommonQwen3_5ForCausalLMBase, +) from vllm.model_executor.models.qwen3_5 import ( Qwen3_5MoeProcessingInfo, Qwen3_5ProcessingInfo, @@ -647,6 +650,7 @@ class Qwen3_5ForCausalLMBase( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, ): packed_modules_mapping = { "qkv_proj": [ @@ -733,33 +737,17 @@ def forward( def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config( + vllm_config ) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, + ) -> MambaStateShapes: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_shape_from_config( + vllm_config ) @classmethod @@ -815,7 +803,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): info=Qwen3_5ProcessingInfo, dummy_inputs=Qwen3VLDummyInputsBuilder, ) -class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid): +class Qwen3_5ForConditionalGeneration( + Qwen3VLForConditionalGeneration, IsHybrid, SupportsReplaySSM +): supports_multimodal_pruning = True hf_to_vllm_mapper = ( @@ -937,33 +927,17 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_dtype_from_config( + vllm_config ) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig" - ) -> tuple[tuple[int, int], tuple[int, int, int]]: - parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_text_config - tp_size = parallel_config.tensor_parallel_size - num_spec = ( - vllm_config.speculative_config.num_speculative_tokens - if vllm_config.speculative_config - else 0 - ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( - tp_size, - hf_config.linear_num_key_heads, - hf_config.linear_num_value_heads, - hf_config.linear_key_head_dim, - hf_config.linear_value_head_dim, - hf_config.linear_conv_kernel_dim, - num_spec, + ) -> MambaStateShapes: + return CommonQwen3_5ForCausalLMBase.get_mamba_state_shape_from_config( + vllm_config ) @classmethod diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index f705c9c76705..eb276691dcd5 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -34,12 +34,14 @@ from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, + MambaStateShapes, MixtureOfExperts, MultiModalEmbeddings, SupportsHCSP, SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, _require_is_multimodal, ) from vllm.model_executor.models.qwen3_5 import ( @@ -722,6 +724,7 @@ class Qwen4ExpForCausalLM( SupportsLoRA, SupportsMRoPE, SupportsPP, + SupportsReplaySSM, Qwen4ExpMixtureOfExperts, IsHybrid, ): @@ -844,17 +847,24 @@ def get_ple_mamba_state_shape_from_config( @classmethod def get_gdn_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig - ) -> tuple[torch.dtype, torch.dtype]: - return MambaStateDtypeCalculator.gated_delta_net_state_dtype( - vllm_config.model_config.dtype, - vllm_config.cache_config.mamba_cache_dtype, - vllm_config.cache_config.mamba_ssm_cache_dtype, + ) -> tuple[torch.dtype, ...]: + dtypes: tuple[torch.dtype, ...] = ( + MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) ) + if vllm_config.is_gdn_replayssm_enabled(): + dtypes += MambaStateDtypeCalculator.gated_delta_net_replayssm_ring_dtypes( + vllm_config.model_config.dtype + ) + return dtypes @classmethod def get_gdn_mamba_state_shape_from_config( cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: parallel_config = vllm_config.parallel_config hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size @@ -863,7 +873,7 @@ def get_gdn_mamba_state_shape_from_config( if vllm_config.speculative_config else 0 ) - return MambaStateShapeCalculator.gated_delta_net_state_shape( + shapes = MambaStateShapeCalculator.gated_delta_net_state_shape( tp_size, hf_config.linear_num_key_heads, hf_config.linear_num_value_heads, @@ -872,18 +882,27 @@ def get_gdn_mamba_state_shape_from_config( hf_config.linear_conv_kernel_dim, num_spec, ) + if vllm_config.is_gdn_replayssm_enabled(): + shapes += MambaStateShapeCalculator.gated_delta_net_replayssm_ring_shapes( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + ) + return shapes @classmethod def get_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: + ) -> tuple[torch.dtype, ...]: return cls.get_gdn_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: VllmConfig - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: return cls.get_gdn_mamba_state_shape_from_config(vllm_config) @classmethod @@ -1160,14 +1179,14 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def get_mamba_state_dtype_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[torch.dtype, torch.dtype]: + ) -> tuple[torch.dtype, ...]: return Qwen4ExpForCausalLM.get_mamba_state_dtype_from_config(vllm_config) @classmethod def get_mamba_state_shape_from_config( cls, vllm_config: VllmConfig, - ) -> tuple[tuple[int, int], tuple[int, int]]: + ) -> MambaStateShapes: return Qwen4ExpForCausalLM.get_mamba_state_shape_from_config(vllm_config) @classmethod diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index 45e576311533..a2eef680c3c1 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -171,6 +171,8 @@ def _make_draft_vllm_config( draft_vllm_config = replace( vllm_config, model_config=speculative_config.draft_model_config, + # Draft layers use attention, without GDN or PLE replay state. + cache_config=replace(vllm_config.cache_config, use_replayssm=False), ) # VllmConfig post-init derives the target quant config, so restore the # independently resolved draft quant config after replacement. diff --git a/vllm/ray/ray_env.py b/vllm/ray/ray_env.py index c3b3de07c0a2..16f75a00048a 100644 --- a/vllm/ray/ray_env.py +++ b/vllm/ray/ray_env.py @@ -44,6 +44,7 @@ } DEFAULT_EXTRA_ENV_VARS: set[str] = { + "GDN_UCACHE_STATE_DTYPE", "PYTHONHASHSEED", } diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 5f707ad025ae..2a3313115078 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -455,6 +455,11 @@ class CommonAttentionMetadata: (num_computed_tokens < num_prompt_tokens). Used by some backends to distinguish actual decodes from short extends.""" + is_last_prefill: torch.Tensor | None = None + """(batch_size,) bool tensor: True if this scheduled chunk reaches the end + of a request's prompt. GDN ReplaySSM replays such a chunk as a decode row + when it is decode-shaped; earlier chunks must update the canonical state.""" + seq_lens_cpu_upper_bound: torch.Tensor | None = None """(batch_size,) CPU upper bound on seq_lens. Precise for prefill rows and for all rows outside async spec decode; optimistic for async-spec @@ -548,6 +553,7 @@ def unpadded( seq_lens_cpu_upper_bound=maybe_slice_reqs(self.seq_lens_cpu_upper_bound), is_prefilling=maybe_slice_reqs(self.is_prefilling), req_idx=maybe_slice_reqs(self.req_idx), + is_last_prefill=maybe_slice_reqs(self.is_last_prefill), rswa_prefix_lens=maybe_slice_reqs(self.rswa_prefix_lens), ) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index adc401f73434..ea74efc1bd91 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -14,6 +14,7 @@ MambaPrefillCheckpointBuilder, MambaPrefillCheckpointMetadata, ) +from vllm.model_executor.layers.mamba.mamba_utils import gdn_replayssm_executed_width from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -90,6 +91,28 @@ def supports_device_cpu_query_lens_mismatch(cls) -> bool: return True +def gdn_replayssm_prefill_rows( + *, + is_prefilling: torch.Tensor, + is_last_prefill: torch.Tensor, + has_prior_state: torch.Tensor, + query_lens: torch.Tensor, + num_draft_tokens: torch.Tensor | None, +) -> torch.Tensor: + """Return the rows that GDN ReplaySSM must run as canonical prefill. + + Prompt chunks run as prefill, except the last chunk of a prompt that + already has state when it is shaped like a decode row: one token, or one + token plus its draft tokens. That chunk replays instead. Inputs are CPU tensors. + """ + decode_shaped = query_lens == 1 + if num_draft_tokens is not None: + decode_shaped = decode_shaped | ( + (num_draft_tokens >= 0) & (query_lens == num_draft_tokens + 1) + ) + return is_prefilling & ~(is_last_prefill & has_prior_state & decode_shaped) + + @dataclass class GDNAttentionMetadata: num_prefills: int @@ -121,6 +144,11 @@ class GDNAttentionMetadata: num_accepted_tokens: torch.Tensor | None = None # shape: [batch,] uniform_spec_sequence_length: int | None = None # None for ragged batches + # GDN ReplaySSM routes every replayed decode row, with or without draft + # tokens, through the spec_* fields above. Its kernel pads each row to the + # launch width; these indices select the real-token outputs. + replayssm_output_indices: torch.Tensor | None = None + # Pre-computed FLA chunk metadata (avoids GPU->CPU sync in prepare_chunk_indices) chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -192,6 +220,12 @@ def __init__( else: self.num_spec = 0 self.use_spec_decode: bool = self.num_spec > 0 + self.use_gdn_replayssm = vllm_config.is_gdn_replayssm_enabled() + self.replayssm_executed_width = ( + gdn_replayssm_executed_width(self.num_spec) + if self.use_gdn_replayssm + else None + ) self._init_reorder_batch_threshold(1, self.use_spec_decode) self.use_full_cuda_graph: bool = ( @@ -217,8 +251,11 @@ def __init__( self.compilation_config.max_cudagraph_capture_size, ) + # ReplaySSM uses one canonical state plus replay records. The ordinary + # speculative path keeps a separate state index for each token. + self.state_table_width = 1 if self.use_gdn_replayssm else self.num_spec + 1 self.spec_state_indices_tensor: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs, self.num_spec + 1), + (self.decode_cudagraph_max_bs, self.state_table_width), dtype=torch.int32, device=device, ) @@ -257,6 +294,11 @@ def __init__( dtype=torch.int32, device=device, ) + self.replayssm_output_indices: torch.Tensor | None = None + if self.use_gdn_replayssm: + self.replayssm_output_indices = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device + ) @staticmethod def _check_chunk_metadata_override( @@ -352,7 +394,45 @@ def build( # type: ignore[override] uniform_spec_sequence_length = None spec_sequence_masks_cpu: torch.Tensor | None = None - if not self.use_spec_decode or num_decode_draft_tokens_cpu is None: + if self.use_gdn_replayssm: + if ( + m.is_prefilling is None + or m.is_last_prefill is None + or m.seq_lens_cpu_upper_bound is None + ): + raise ValueError( + "FlashInfer GDN ReplaySSM requires prefill, final-prompt, and " + "sequence-length metadata" + ) + query_lens_cpu_all = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + assert self.replayssm_executed_width is not None + prefill_rows = gdn_replayssm_prefill_rows( + is_prefilling=m.is_prefilling, + is_last_prefill=m.is_last_prefill, + has_prior_state=m.seq_lens_cpu_upper_bound > query_lens_cpu_all, + query_lens=query_lens_cpu_all, + num_draft_tokens=num_decode_draft_tokens_cpu, + ) + spec_sequence_masks_cpu = ( + ~prefill_rows + & (query_lens_cpu_all > 0) + & (query_lens_cpu_all <= self.replayssm_executed_width) + ) + num_spec_decodes = int(spec_sequence_masks_cpu.sum().item()) + if num_spec_decodes == 0: + spec_sequence_masks = None + spec_sequence_masks_cpu = None + else: + spec_sequence_masks = async_tensor_h2d( + spec_sequence_masks_cpu, device=query_start_loc.device + ) + if num_accepted_tokens is None: + num_accepted_tokens = torch.ones( + m.num_reqs, + dtype=torch.int32, + device=query_start_loc.device, + ) + elif not self.use_spec_decode or num_decode_draft_tokens_cpu is None: spec_sequence_masks = None num_spec_decodes = 0 else: @@ -370,6 +450,7 @@ def build( # type: ignore[override] spec_sequence_masks_cpu, device=query_start_loc.device ) + replayssm_output_indices = None if spec_sequence_masks is None: # V2 already excludes prefills from full decode graphs via # has_prefill. Classify first chunks as prefills to mask recycled @@ -385,9 +466,13 @@ def build( # type: ignore[override] no_prior_state &= m.is_prefilling else: no_prior_state = torch.zeros_like(no_prior_state) + # Under GDN ReplaySSM every decode row replays above, so the rows + # here keep the scheduler's prefill flag. num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( - m.replace(is_prefilling=no_prior_state), + m + if self.use_gdn_replayssm + else m.replace(is_prefilling=no_prior_state), decode_threshold=1, treat_short_extends_as_decodes=False, ) @@ -446,9 +531,14 @@ def build( # type: ignore[override] num_decode_tokens = 0 if num_prefills == 0 and num_decodes == 0: - spec_token_size = min( - num_spec_decodes * (self.num_spec + 1), - query_start_loc_cpu[-1].item(), + # Replay rows can be shorter than num_spec + 1 tokens. + spec_token_size = ( + query_start_loc_cpu[-1].item() + if self.use_gdn_replayssm + else min( + num_spec_decodes * (self.num_spec + 1), + query_start_loc_cpu[-1].item(), + ) ) spec_token_indx = torch.arange( spec_token_size, @@ -461,7 +551,7 @@ def build( # type: ignore[override] # Padded sequences trail the spec decodes, so slice them off # rather than gather with the host mask (an H2D copy + a kernel). spec_state_indices_tensor = block_table_tensor[ - :num_spec_decodes, : self.num_spec + 1 + :num_spec_decodes, : self.state_table_width ] non_spec_state_indices_tensor = None # Padded sequences are always at the back, so the first @@ -482,7 +572,7 @@ def build( # type: ignore[override] spec_token_indx = index[num_non_spec_tokens:] spec_state_indices_tensor = block_table_tensor[ - spec_sequence_masks_cpu, : self.num_spec + 1 + spec_sequence_masks_cpu, : self.state_table_width ] non_spec_state_indices_tensor = block_table_tensor[ non_spec_sequence_masks_cpu, 0 @@ -523,6 +613,21 @@ def build( # type: ignore[override] num_accepted_tokens = num_accepted_tokens[:num_spec_decodes] else: num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] + if self.use_gdn_replayssm: + assert self.replayssm_executed_width is not None + replay_query_lens_cpu = query_lens_cpu[spec_sequence_masks_cpu] + row_offsets = torch.arange( + self.replayssm_executed_width, dtype=torch.int32 + ).unsqueeze(0) + dense_indices = ( + torch.arange(num_spec_decodes, dtype=torch.int32).unsqueeze(1) + * self.replayssm_executed_width + + row_offsets + ) + replayssm_output_indices = async_tensor_h2d( + dense_indices[row_offsets < replay_query_lens_cpu.unsqueeze(1)], + device=query_start_loc.device, + ) chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -670,6 +775,18 @@ def build( # type: ignore[override] num_accepted_tokens = self.num_accepted_tokens[:batch_size] num_accepted_tokens[num_spec_decodes:].fill_(1) + if self.use_gdn_replayssm: + assert replayssm_output_indices is not None + assert self.replayssm_output_indices is not None + output_count = replayssm_output_indices.numel() + self.replayssm_output_indices[:output_count].copy_( + replayssm_output_indices, non_blocking=True + ) + # A captured graph keeps its capture-time view length; zero the + # stale tail so its extra indices stay in bounds. + self.replayssm_output_indices[output_count:].zero_() + replayssm_output_indices = self.replayssm_output_indices[:output_count] + if self._stage_decode(num_prefills, num_decodes, num_spec_decodes): self.non_spec_state_indices_tensor[:num_decodes].copy_( non_spec_state_indices_tensor, non_blocking=True @@ -713,6 +830,7 @@ def build( # type: ignore[override] non_spec_token_indx=non_spec_token_indx, num_accepted_tokens=num_accepted_tokens, uniform_spec_sequence_length=uniform_spec_sequence_length, + replayssm_output_indices=replayssm_output_indices, nums_dict=nums_dict, batch_ptr=batch_ptr, token_chunk_offset_ptr=token_chunk_offset_ptr, @@ -771,9 +889,9 @@ def update_block_table( prefill_indices = non_spec_indices[m.num_decodes :] elif m.num_prefills == 0: # Same as build(): padded sequences trail the spec decodes. - spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1] + spec_indices = blk_table[: m.num_spec_decodes, : self.state_table_width] else: - spec_indices = blk_table[masks, : self.num_spec + 1] + spec_indices = blk_table[masks, : self.state_table_width] non_spec_indices = prefill_indices = blk_table[~masks, 0] if self._stage_spec_decode( diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 1bace4dab7b5..d454667ec459 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -15,6 +15,7 @@ AttentionMetadataBuilder, CommonAttentionMetadata, ) +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.utils import ( NULL_BLOCK_ID, compute_causal_conv1d_metadata, @@ -107,9 +108,14 @@ def __init__( self.compilation_config = vllm_config.compilation_config self.num_spec_tokens: int = vllm_config.num_speculative_tokens self.use_spec_decode = self.num_spec_tokens > 0 - self.use_replayssm = vllm_config.cache_config.use_replayssm + self.use_replayssm = ( + vllm_config.cache_config.use_replayssm + and kv_cache_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2 + ) self.replayssm_buffer_len = vllm_config.cache_config.replayssm_buffer_len - self.use_flashinfer_replayssm = vllm_config.use_flashinfer_replayssm + self.use_flashinfer_replayssm = ( + self.use_replayssm and vllm_config.use_flashinfer_replayssm + ) scheduler_config = vllm_config.scheduler_config self.decode_cudagraph_max_bs: int = scheduler_config.max_num_seqs diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9df68b86e989..f1fbfe8140ef 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1425,6 +1425,12 @@ def gather_batch_req_state( if num_draft_tokens_np is not None: num_new_tokens = num_new_tokens - num_draft_tokens_np one_token_ext = (num_new_tokens == 1) & (num_computed_prefill_tokens_np > 0) + if self.vllm_config.is_gdn_replayssm_enabled(): + # Intermediate GDN chunks must update the canonical state before + # a later multi-token prefill reads it. + one_token_ext &= ( + num_computed_prefill_tokens_np + num_new_tokens >= prefill_len_np + ) prefill_runs_as_decode_np = is_prefilling_np & one_token_ext decode_graph_eligible = bool( (prefill_runs_as_decode_np | ~is_prefilling_np).all() diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index c0205c7c0b0f..0b52fca39071 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -12,7 +12,10 @@ from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFuncsByType from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d -from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder +from vllm.v1.attention.backends.gdn_attn import ( + GDNAttentionMetadata, + GDNAttentionMetadataBuilder, +) from vllm.v1.attention.backends.mamba2_attn import ( Mamba2AttentionMetadata, Mamba2AttentionMetadataBuilder, @@ -37,20 +40,27 @@ preprocess_mamba_align_fused_kernel, validate_mamba_state_copy_funcs, ) -from vllm.v1.worker.replayssm_utils import initialize_replayssm_ring_trackers +from vllm.v1.worker.replayssm_utils import ( + initialize_replayssm_ring_trackers, + may_cross_mamba_block_boundary, +) from vllm.v1.worker.utils import AttentionGroup @dataclass class MambaHybridAttnMetadata(ModelSpecificAttnMetadata): is_prefilling: torch.Tensor + is_last_prefill: torch.Tensor num_accepted_tokens: torch.Tensor | None = None num_decode_draft_tokens_cpu: torch.Tensor | None = None def get_extra_common_attn_kwargs( self, kv_cache_group_id: int, num_reqs: int ) -> dict[str, Any]: - return {"is_prefilling": self.is_prefilling[:num_reqs]} + return { + "is_prefilling": self.is_prefilling[:num_reqs], + "is_last_prefill": self.is_last_prefill[:num_reqs], + } def get_extra_attn_kwargs( self, attn_metadata_builder: Any, num_reqs: int @@ -91,12 +101,14 @@ def __init__( self.max_num_reqs, dtype=torch.int32, device=self.device ) self._replayssm_query_start_loc: torch.Tensor | None = None + self._replayssm_may_materialize = True # Pre-copy prefix-cache state (V2). The migration of each request's # mamba state across block boundaries runs as a fused GPU kernel reusing # the postprocess copy machinery, so the per-step src columns and the # running state_idx are kept GPU-resident. self._align_mode = self.cache_config.mamba_cache_mode == "align" self._use_flashinfer_replayssm = vllm_config.use_flashinfer_replayssm + self._use_gdn_replayssm = vllm_config.is_gdn_replayssm_enabled() if self._use_flashinfer_replayssm: self._is_prefilling_gpu = torch.zeros( self.max_num_reqs, dtype=torch.bool, device=self.device @@ -286,6 +298,15 @@ def prepare_attn( is_prefilling_np = is_prefilling_np & ~input_batch.prefill_runs_as_decode_np is_prefilling = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") is_prefilling[: input_batch.num_reqs] = torch.from_numpy(is_prefilling_np) + is_last_prefill = torch.zeros(num_reqs, dtype=torch.bool, device="cpu") + is_last_prefill[: input_batch.num_reqs] = torch.from_numpy( + input_batch.is_prefilling_np + & ( + input_batch.num_computed_prefill_tokens_np + + input_batch.num_scheduled_tokens + >= input_batch.prefill_len_np + ) + ) # During CUDAGraph capture, num_decode_draft_tokens_cpu and num_accepted_tokens # are created by attn_metadata_builder.build_for_cudagraph_capture, so we only # compute them during actual (non-capture) forward execution. @@ -338,6 +359,7 @@ def prepare_attn( mamba_attn_metadata = MambaHybridAttnMetadata( is_prefilling=is_prefilling, + is_last_prefill=is_last_prefill, num_accepted_tokens=num_accepted_tokens, num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) @@ -362,20 +384,52 @@ def prepare_attn( ) if self._use_flashinfer_replayssm: self._replayssm_query_start_loc = None + self._replayssm_may_materialize = True + if ( + self._use_gdn_replayssm + and self._align_mode + and self.vllm_config.num_speculative_tokens == 0 + ): + assert self._mamba_ctx is not None + replay = self._mamba_ctx.replayssm + assert replay is not None + # Skip GDN's per-layer materialization when no request can reach + # a block boundary. With draft tokens the CPU positions here are + # only upper bounds, so the gate applies to standard decoding. + self._replayssm_may_materialize = may_cross_mamba_block_boundary( + input_batch.num_computed_tokens_np, + input_batch.num_scheduled_tokens, + replay.mamba_block_size, + ) + metadata_type = ( + GDNAttentionMetadata + if self._use_gdn_replayssm + else Mamba2AttentionMetadata + ) mamba_metadata = next( ( metadata for metadata in attn_metadata.values() - if isinstance(metadata, Mamba2AttentionMetadata) + if isinstance(metadata, metadata_type) ), None, ) # Autotune dummy runs and draft capture can omit Mamba metadata. if mamba_metadata is not None: self._replayssm_query_start_loc = input_batch.query_start_loc - replayssm_prefilling = ( - torch.arange(num_reqs, device="cpu") >= mamba_metadata.num_decodes - ) + if ( + isinstance(mamba_metadata, GDNAttentionMetadata) + and mamba_metadata.spec_sequence_masks_cpu is not None + ): + # GDN replay rows are the spec rows; all others prefilled. + replayssm_prefilling = ~mamba_metadata.spec_sequence_masks_cpu[ + :num_reqs + ] + else: + replayssm_prefilling = ( + torch.arange(num_reqs, device="cpu") + >= mamba_metadata.num_decodes + ) async_tensor_h2d( replayssm_prefilling, out=self._is_prefilling_gpu[:num_reqs] ) @@ -483,6 +537,7 @@ def _publish_flashinfer_replayssm( is_prefilling=self._is_prefilling_gpu[:num_reqs], live_cols=(self._mamba_state_idx_gpu if self._align_mode else None), num_reqs=num_reqs, + may_materialize=self._replayssm_may_materialize, ) diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index a1cb281a2bfe..e71e22ed3e4a 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -707,13 +707,17 @@ def _get_mamba_gpu_state_copy_funcs( state_copy_funcs = copy_funcs[spec.mamba_type] if isinstance(layer, MambaBase) and layer.use_flashinfer_replayssm: if ( - spec.mamba_type != MambaAttentionBackendEnum.MAMBA2 + spec.mamba_type + not in ( + MambaAttentionBackendEnum.MAMBA2, + MambaAttentionBackendEnum.GDN_ATTN, + ) or len(spec.shapes) != 5 or len(spec.dtypes) != 5 or state_copy_funcs != (get_conv_copy_spec, get_temporal_copy_spec) ): raise ValueError( - "FlashInfer ReplaySSM requires two canonical Mamba2 " + "FlashInfer ReplaySSM requires two canonical Mamba2 or GDN " "states and three ring states" ) state_copy_funcs += (get_temporal_copy_spec,) * 3 @@ -933,7 +937,7 @@ def _populate_metadata( f"expected {self.num_groups} block tables, got {len(block_tables)}" ) replayssm_groups: list[tuple[list[Any], MambaSpec, torch.Tensor]] = [] - has_baseline_layer = False + has_baseline_temporal_state = False for group_local_idx, mamba_group_id in enumerate(self.mamba_group_ids): kv_cache_group = kv_cache_config.kv_cache_groups[mamba_group_id] mixers: list[Any] = [] @@ -949,11 +953,15 @@ def _populate_metadata( isinstance(attention, MambaBase) and attention.use_flashinfer_replayssm ) - has_baseline_layer |= not is_flashinfer_replayssm + has_baseline_temporal_state |= ( + not is_flashinfer_replayssm + and get_temporal_copy_spec in state_copy_funcs + ) if is_flashinfer_replayssm: if len(kv_caches) != len(state_copy_funcs): raise ValueError( - "FlashInfer ReplaySSM requires five bound Mamba2 states" + "FlashInfer ReplaySSM requires five bound Mamba2 or GDN " + "states" ) mixers.append(attention) if len(kv_caches) < len(state_copy_funcs): @@ -1041,20 +1049,20 @@ def _populate_metadata( idx += 1 if mixers: - spec = kv_cache_group.kv_cache_spec - if not isinstance(spec, MambaSpec): - raise TypeError( - "FlashInfer ReplaySSM layers require a Mamba cache spec; " - f"got {type(spec).__name__}" - ) + spec = _get_mamba_spec_for_layer(kv_cache_group, layer_names[0]) + assert all( + _get_mamba_spec_for_layer(kv_cache_group, name) == spec + for name in layer_names + ), "FlashInfer ReplaySSM requires equal Mamba cache specs" replayssm_groups.append( (mixers, spec, aligned_index_block_tables[group_local_idx]) ) assert idx == self.num_states - if replayssm_groups and has_baseline_layer: + if replayssm_groups and has_baseline_temporal_state: raise ValueError( - "mixed FlashInfer ReplaySSM and baseline Mamba layers are unsupported" + "mixed FlashInfer ReplaySSM and baseline Mamba temporal states " + "are unsupported" ) # Cache per-group block-table base addresses and per-request stride. diff --git a/vllm/v1/worker/replayssm_utils.py b/vllm/v1/worker/replayssm_utils.py index 4570b3413a8a..8725ac86a5a9 100644 --- a/vllm/v1/worker/replayssm_utils.py +++ b/vllm/v1/worker/replayssm_utils.py @@ -7,15 +7,32 @@ from functools import cache from typing import Any, NamedTuple +import numpy as np import torch from vllm.model_executor.layers.mamba.abstract import MambaBase +from vllm.model_executor.layers.mamba.gdn.replayssm import ( + load_gdn_replayssm_materializer, +) +from vllm.model_executor.layers.mamba.mamba_utils import GDN_REPLAYSSM_RING_SLOTS from vllm.model_executor.models.utils import extract_layer_index from vllm.triton_utils import tl, triton +from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +def may_cross_mamba_block_boundary( + computed: np.ndarray, scheduled: np.ndarray, block_size: int +) -> bool: + """Whether any request could cross a Mamba block boundary this step. + + A request accepts at most its scheduled tokens, so `False` means no prefix + snapshot can be due and postprocess may skip materialization. + """ + return bool(((computed + scheduled) // block_size > computed // block_size).any()) + + def initialize_replayssm_ring_trackers( kv_cache_config: KVCacheConfig, forward_context: Mapping[str, Any], @@ -174,6 +191,7 @@ def _postprocess_replayssm_kernel( HAS_IDX_MAPPING: tl.constexpr, MATERIALIZE_PREFIXES: tl.constexpr, LIVE_COL_IS_ZERO: tl.constexpr, + EXECUTED_QUERY_WIDTH: tl.constexpr, ) -> None: """Commit a completed step and prepare an optional prefix snapshot.""" batch_idx = tl.program_id(0) @@ -213,7 +231,8 @@ def _postprocess_replayssm_kernel( NUM_COMPUTED_IS_POST_STEP, computed - query_len, computed ) # Staged from forward metadata using Mamba attention's classification. - prefilling = tl.load(is_prefilling + batch_idx) + valid_query = query_len > 0 + prefilling = tl.load(is_prefilling + batch_idx) & valid_query accepted = tl.maximum(tl.load(num_accepted_tokens + req_idx), 1) # Derive this request's pre/post-step positions from ReplaySSM metadata. @@ -283,10 +302,15 @@ def _postprocess_replayssm_kernel( if wants_materialize & (live_col != dst_col): # Prefill already wrote canonical state; only another slot needs a copy. tl.store(plan_flush_count + batch_idx, 0, mask=writes_plan) - elif valid_live: + elif valid_live & valid_query: old_start = tl.load(tracker_start + live_slot) old_committed = tl.load(tracker_committed + live_slot) - checkpointed = old_committed + query_len > LOGICAL_WINDOW + # GDN launches pad every row to a fixed width, and its kernel decides + # whether to fold from that width rather than from this row's length. + executed_query_len = query_len + if EXECUTED_QUERY_WIDTH > 0: + executed_query_len = EXECUTED_QUERY_WIDTH + checkpointed = old_committed + executed_query_len > LOGICAL_WINDOW next_start = tl.where( checkpointed, (old_start + old_committed) % RING_BUFFER_LEN, @@ -315,6 +339,7 @@ def _postprocess_replayssm_kernel( def _compact_replayssm_requests_kernel( plan_flush_count, active_request_indices, + num_active_ptr, num_reqs, MAX_NUM_REQS: tl.constexpr, BLOCK_SIZE: tl.constexpr, @@ -328,6 +353,7 @@ def _compact_replayssm_requests_kernel( active_i32 = active.to(tl.int32) output_offsets = tl.cumsum(active_i32, axis=0) - 1 num_active = tl.sum(active_i32, axis=0) + tl.store(num_active_ptr, num_active) tl.store( active_request_indices + output_offsets, offsets, @@ -367,17 +393,20 @@ class ReplaySSMModelContext: src_slots_ptrs: torch.Tensor dst_slots_ptrs: torch.Tensor num_layers: torch.Tensor - materialize_tables: _ReplaySSMMaterializeTables + materialize_tables: _ReplaySSMMaterializeTables | None src_slots: torch.Tensor dst_slots: torch.Tensor plan_ring_start: torch.Tensor plan_flush_count: torch.Tensor active_request_indices: torch.Tensor + num_active: torch.Tensor max_num_reqs: int mamba_block_size: int logical_window: int ring_buffer_len: int materialize_prefixes: bool + executed_query_width: int # GDN launch width; 0 uses each row's length. + mamba_type: MambaAttentionBackendEnum @classmethod def create( @@ -388,6 +417,7 @@ def create( mixers = [mixer for group, _, _ in grouped for mixer in group] first = mixers[0] first_spec = grouped[0][1] + mamba_type = first_spec.mamba_type # Homogeneous groups share cache-hit lookup, logical columns, and replay # transitions. The scheduler still assigns each group its own slots. assert all(spec == first_spec for _, spec, _ in grouped), ( @@ -396,12 +426,24 @@ def create( geometry = [(state.shape[1:], state.dtype) for state in first.kv_cache[1:]] assert all( [(state.shape[1:], state.dtype) for state in mixer.kv_cache[1:]] == geometry - and mixer.A.dtype == first.A.dtype + and ( + mamba_type == MambaAttentionBackendEnum.GDN_ATTN + or mixer.A.dtype == first.A.dtype + ) and mixer.replayssm_buffer_len == first.replayssm_buffer_len for mixer in mixers ), "FlashInfer ReplaySSM requires equal materialization geometry" device = first.kv_cache[1].device + executed_query_width = 0 + if mamba_type == MambaAttentionBackendEnum.GDN_ATTN: + # Every GDN layer derives its launch width from the same config. + executed_query_width = first.replayssm_executed_width + if first.kv_cache[2].size(2) != GDN_REPLAYSSM_RING_SLOTS: + raise ValueError( + f"FlashInfer GDN ReplaySSM requires {GDN_REPLAYSSM_RING_SLOTS} " + f"ring slots, got {first.kv_cache[2].size(2)}" + ) block_tables = [table for _, _, table in grouped] src_slots = torch.full( (len(mixers), max_num_reqs), @@ -415,7 +457,30 @@ def create( layer_offsets.append(layer_offsets[-1] + len(group)) # Each pointer selects a group's first row in the flat [layer, batch] # allocation. Different group sizes need no padding or per-step concat. - zero_table = torch.zeros(len(mixers), dtype=torch.int64, device=device) + materialize_tables = None + if mamba_type != MambaAttentionBackendEnum.GDN_ATTN: + zero_table = torch.zeros(len(mixers), dtype=torch.int64, device=device) + materialize_tables = _ReplaySSMMaterializeTables( + state_ptrs=_cuda_i64_ptrs([m.kv_cache[1] for m in mixers]), + state_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[1] for m in mixers] + ), + x_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[2] for m in mixers]), + x_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[2] for m in mixers] + ), + B_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[4] for m in mixers]), + B_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[4] for m in mixers] + ), + dt_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[3] for m in mixers]), + dt_cache_slot_strides=_cuda_i64_slot_strides( + [m.kv_cache[3] for m in mixers] + ), + A_ptrs=_cuda_i64_ptrs([m.A for m in mixers]), + state_scale_ptrs=zero_table, + state_scale_slot_strides=zero_table, + ) return cls( mixers=mixers, block_tables=block_tables, @@ -438,27 +503,7 @@ def create( dtype=torch.int32, device=device, ), - materialize_tables=_ReplaySSMMaterializeTables( - state_ptrs=_cuda_i64_ptrs([m.kv_cache[1] for m in mixers]), - state_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[1] for m in mixers] - ), - x_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[2] for m in mixers]), - x_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[2] for m in mixers] - ), - B_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[4] for m in mixers]), - B_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[4] for m in mixers] - ), - dt_cache_ptrs=_cuda_i64_ptrs([m.kv_cache[3] for m in mixers]), - dt_cache_slot_strides=_cuda_i64_slot_strides( - [m.kv_cache[3] for m in mixers] - ), - A_ptrs=_cuda_i64_ptrs([m.A for m in mixers]), - state_scale_ptrs=zero_table, - state_scale_slot_strides=zero_table, - ), + materialize_tables=materialize_tables, src_slots=src_slots, dst_slots=dst_slots, plan_ring_start=torch.zeros(max_num_reqs, dtype=torch.int32, device=device), @@ -468,6 +513,9 @@ def create( active_request_indices=torch.full( (max_num_reqs,), -1, dtype=torch.int32, device=device ), + num_active=torch.zeros(1, dtype=torch.int32, device=device), + executed_query_width=executed_query_width, + mamba_type=mamba_type, max_num_reqs=max_num_reqs, mamba_block_size=first_spec.block_size, logical_window=int(first.replayssm_buffer_len), @@ -476,7 +524,15 @@ def create( ) def _materialize(self) -> None: - """Publish all canonical prefix snapshots in one FlashInfer call.""" + """Publish canonical prefix snapshots. + + Mamba2 publishes every layer in one FlashInfer call. FlashInfer's GDN + materializer takes one layer per call. + """ + if self.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: + self._materialize_gdn() + return + assert self.materialize_tables is not None first = self.mixers[0] mamba_config = first.mamba_config rand_seed = None @@ -507,6 +563,23 @@ def _materialize(self) -> None: philox_rounds=philox_rounds, ) + def _materialize_gdn(self) -> None: + materialize_fn = load_gdn_replayssm_materializer() + for layer_idx, mixer in enumerate(self.mixers): + u_cache, k_cache, g_cache = mixer.kv_cache[2:5] + materialize_fn( + state=mixer.kv_cache[1], + src_slots=self.src_slots[layer_idx], + dst_slots=self.dst_slots[layer_idx], + k_cache=k_cache, + u_cache=u_cache, + g_cache=g_cache, + cache_base=self.plan_ring_start, + count=self.plan_flush_count, + active_request_indices=self.active_request_indices, + num_active=self.num_active, + ) + def reset_new_slots( self, *, @@ -542,6 +615,7 @@ def postprocess( is_prefilling: torch.Tensor, live_cols: torch.Tensor | None, num_reqs: int, + may_materialize: bool = True, ) -> None: """Commit all groups before publishing their canonical prefix states.""" if num_reqs == 0: @@ -572,12 +646,14 @@ def postprocess( HAS_IDX_MAPPING=idx_mapping is not None, MATERIALIZE_PREFIXES=self.materialize_prefixes, LIVE_COL_IS_ZERO=live_cols is None, + EXECUTED_QUERY_WIDTH=self.executed_query_width, ) - if not self.materialize_prefixes: + if not self.materialize_prefixes or not may_materialize: return _compact_replayssm_requests_kernel[(1,)]( self.plan_flush_count, self.active_request_indices, + self.num_active, num_reqs, MAX_NUM_REQS=self.max_num_reqs, BLOCK_SIZE=triton.next_power_of_2(self.max_num_reqs),