Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions tests/compile/correctness_e2e/test_sequence_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ def _compare_sp(
num_gpus_available: int,
use_inductor_graph_partition: bool,
fuse_gemm_comms: bool,
enable_prompt_embeds: bool,
*,
method: Literal["generate", "encode"],
is_multimodal: bool,
Expand Down Expand Up @@ -248,6 +249,8 @@ def _compare_sp(
"--enable-mm-embeds",
]
)
elif enable_prompt_embeds:
common_args.append("--enable-prompt-embeds")

compilation_config = {
"mode": CompilationMode.VLLM_COMPILE,
Expand All @@ -257,7 +260,9 @@ def _compare_sp(
"fuse_gemm_comms": fuse_gemm_comms,
"fuse_norm_quant": fuse_norm_quant,
"fuse_act_quant": fuse_act_quant,
"fuse_allreduce_rms": False,
"eliminate_noops": True,
"sp_min_token_num": 0,
},
"use_inductor_graph_partition": use_inductor_graph_partition,
}
Expand Down Expand Up @@ -349,6 +354,49 @@ def test_tp_sp_generation(
num_gpus_available,
use_inductor_graph_partition,
fuse_gemm_comms=fuse_gemm_comms,
enable_prompt_embeds=False,
method="generate",
is_multimodal=False,
)


# Focused regression test for the SP + prompt_embeds graph-rewrite path.
# Covers pp_size=1 (SP only) and pp_size=2 (SP + PP); kept small on purpose so
# we don't double the matrix of `test_tp_sp_generation` above.
SP_PROMPT_EMBEDS_PARALLEL_SETUPS = [
ParallelSetup(
tp_size=2,
pp_size=pp_size,
fuse_norm_quant=False,
fuse_act_quant=False,
eager_mode=False,
chunked_prefill=False,
)
for pp_size in [1, 2]
]


@pytest.mark.parametrize("parallel_setup", SP_PROMPT_EMBEDS_PARALLEL_SETUPS)
@pytest.mark.parametrize("use_inductor_graph_partition", [True, False])
@create_new_process_for_each_test()
def test_tp_sp_generation_prompt_embeds(
parallel_setup: ParallelSetup,
num_gpus_available,
use_inductor_graph_partition: bool,
):
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")

_compare_sp(
"hmellor/tiny-random-LlamaForCausalLM",
parallel_setup,
distributed_backend="mp",
runner="auto",
test_options=SPTestOptions(multi_node_only=False, load_format=None),
num_gpus_available=num_gpus_available,
use_inductor_graph_partition=use_inductor_graph_partition,
fuse_gemm_comms=False,
enable_prompt_embeds=True,
method="generate",
is_multimodal=False,
)
75 changes: 55 additions & 20 deletions vllm/compilation/passes/fusion/sequence_parallelism.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
from vllm.config import VllmConfig
from vllm.config.utils import Range
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
Expand Down Expand Up @@ -117,6 +120,7 @@ def __init__(
self.device = device
self.tp_group = get_tp_group()
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()

def _all_reduce(self, x: torch.Tensor) -> torch.Tensor:
return tensor_model_parallel_all_reduce(x)
Expand Down Expand Up @@ -204,17 +208,35 @@ def replacement(
mm_1: torch.Tensor,
rms_norm_weights: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
# pattern matcher replaces from top-to-bottom,
# so residual is still the full size here.
# once the seqpar pattern with the previous rmsnorm is replaced
# The pattern matcher replaces from the end of the graph
# (last layer first). At the time each match is replaced,
# the preceding layer has NOT been replaced yet, so
# `residual` is still full-size and the slice below is
# correct. Once the preceding layer IS replaced, its
# residual output shrinks to [local_len, H], and this
# slice becomes semantically incorrect (e.g. for rank > 0,
# the indices would be out of bounds). However, since the
# symbolic output shape equals the input shape,
# NoOpEliminationPass (called at the end of
# SequenceParallelismPass.__call__) removes these slices
# before the graph is ever executed or compiled.
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
local_len = reduce_scatter.size(0)
# when the preceding VocabParallelEmbedding is excluded
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
# the FirstAllReduceRMSNorm pattern is never matched. we must
# perform a proper TP-aware slice here. simply using `[0:local_len]`
# would incorrectly cause all ranks to process rank 0's chunk.
residual = residual[
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
]
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
all_gather = self._all_gather(rmsnorm[0])
# shape of residual changes but that's fine,
# next node is already slicing it, now becomes a noop
# residual output is now [local_len, H]; the next layer's
# slice on it is semantically incorrect until
# NoOpEliminationPass removes it.
return all_gather, rmsnorm[1]

pm.register_replacement(
Expand Down Expand Up @@ -304,19 +326,29 @@ def replacement(
rms_norm_weights: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
# pattern matcher replaces from top-to-bottom,
# so residual is still the full size here.
# add a temporary slice which will become a noop
# once the seqpar pattern with the previous rmsnorm is replaced
# See MiddleAllReduceRMSNormPattern.replacement for a
# detailed explanation of the temporary slice below:
# it is correct when first inserted, becomes semantically
# incorrect after the preceding layer is replaced, and is
# removed by NoOpEliminationPass before the graph is compiled.
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
local_len = reduce_scatter.size(0)
# when the preceding VocabParallelEmbedding is excluded
# from the FX graph (e.g., passing `inputs_embeds` directly in VLMs),
# the FirstAllReduceRMSNorm pattern is never matched. we must
# perform a proper TP-aware slice here. simply using `[0:local_len]`
# would incorrectly cause all ranks to process rank 0's chunk.
residual = residual[
self.tp_rank * local_len : self.tp_rank * local_len + local_len, ...
]
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
quant, _ = self.quant_matcher(rms, scale)
all_gather = self._all_gather(quant)
# shape of residual changes but that's fine,
# next node is already slicing it, now becomes a noop
# residual output is now [local_len, H]; the next layer's
# slice on it is semantically incorrect until
# NoOpEliminationPass removes it.
return all_gather, residual_out

pm.register_replacement(
Expand Down Expand Up @@ -357,12 +389,15 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
gets split across TP ranks, causing size mismatches at subgraph
boundaries.

This pass splits up the residual tensor across TP ranks and hence
divides its size. Because the pattern matcher starts at the end of
the graph, the replacement contains a slice that temporarily conforms
the input residual to the correct size. After all patterns have been
matched, we use a NoOpEliminationPass to clean up what have now
become no-op slices.
This pass splits up the residual tensor across TP ranks and hence divides
its size. The pattern matcher starts at the end of the graph (last layer
first), so when each replacement inserts a residual slice, the preceding
layer has not been replaced yet and the slice is correct. Once the
preceding layer IS replaced, its residual output shrinks and the slice
becomes semantically incorrect (out-of-bounds indices for rank > 0).
The graph is never executed in this intermediate state —
NoOpEliminationPass removes these slices based on symbolic shape equality
(input shape == output shape) before the graph is compiled.
"""

@enable_fake_mode
Expand Down
27 changes: 13 additions & 14 deletions vllm/v1/worker/gpu_model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -3098,7 +3098,7 @@ def get_supported_tasks(self) -> tuple[SupportedTask, ...]:

return tuple(tasks)

def sync_and_slice_intermediate_tensors(
def sync_and_gather_intermediate_tensors(
self,
num_tokens: int,
intermediate_tensors: IntermediateTensors | None,
Expand All @@ -3109,24 +3109,23 @@ def sync_and_slice_intermediate_tensors(
tp = self.vllm_config.parallel_config.tensor_parallel_size
is_rs = is_residual_scattered_for_sp(self.vllm_config, num_tokens)

# When sequence parallelism is enabled, the "residual" tensor is sharded
# across tensor parallel ranks, so each rank only needs its own slice.
# When sequence parallelism is enabled, the "residual" tensor is
# sharded across TP ranks. All-gather it here because downstream
# QKV + Attention needs the full residual before the SP split point.
if sync_self:
assert intermediate_tensors is not None
for k, v in intermediate_tensors.items():
is_scattered = k == "residual" and is_rs
copy_len = num_tokens // tp if is_scattered else num_tokens
self.intermediate_tensors[k][:copy_len].copy_(
v[:copy_len], non_blocking=True
if is_scattered:
local_len = num_tokens // tp
v = get_tp_group().all_gather(v[:local_len], dim=0)

self.intermediate_tensors[k][:num_tokens].copy_(
v[:num_tokens], non_blocking=True
)

return IntermediateTensors(
{
k: v[: num_tokens // tp]
if k == "residual" and is_rs
else v[:num_tokens]
for k, v in self.intermediate_tensors.items()
}
{k: v[:num_tokens] for k, v in self.intermediate_tensors.items()}
)

def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None:
Expand Down Expand Up @@ -3342,7 +3341,7 @@ def _preprocess(
intermediate_tensors = None
else:
assert intermediate_tensors is not None
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
num_input_tokens, intermediate_tensors, True
)

Expand Down Expand Up @@ -5546,7 +5545,7 @@ def _dummy_run(
)
)

intermediate_tensors = self.sync_and_slice_intermediate_tensors(
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
num_tokens_padded, None, False
)

Expand Down
Loading