Skip to content
Open
35 changes: 35 additions & 0 deletions tests/distributed/test_eplb_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
import pytest
import torch

from vllm.distributed.eplb import eplb_state as eplb_state_module
from vllm.distributed.eplb.eplb_state import (
EplbState,
_commit_eplb_maps,
_commit_eplb_maps_for_layer,
)
Expand Down Expand Up @@ -157,3 +159,36 @@ def test_commit_eplb_maps_for_layer():

# Layer 1 untouched
assert torch.equal(model_state.physical_to_logical_map[1], original_phy2log[1])


def test_log_balancedness_from_logical_load(monkeypatch):
"""Per-rank load is the logical load split evenly over each expert's replicas."""
ep_group = MagicMock()
ep_group.device_group.size.return_value = 2
ep_group.device_group.rank.return_value = 0
monkeypatch.setattr(eplb_state_module, "get_ep_group", lambda: ep_group)
logger = MagicMock()
monkeypatch.setattr(eplb_state_module, "logger", logger)
state = EplbState.__new__(EplbState)
state.parallel_config = MagicMock()
state.parallel_config.eplb_config.log_balancedness_interval = 1
state.expert_load_window_size = 1
state.expert_rearrangement_step_interval = 100
state.expert_load_window_step = state.expert_rearrangement_step = 0
state.is_async, state.should_record_tensor = False, None
# Rank 0 holds slots [e0, e1], rank 1 holds [e2, e0].
model_state = _make_model_state(
phy2log=torch.tensor([[0, 1, 2, 0]]),
log2phy=torch.full((1, 3, 2), -1, dtype=torch.long),
logcnt=torch.tensor([[2, 1, 1]]),
)
model_state.expert_load_pass = torch.tensor([[4, 2, 6]])
model_state.expert_load_window = torch.zeros((1, 1, 3), dtype=torch.long)
state.model_states = {"m": model_state}
monkeypatch.setattr(state, "_sync_load_pass", lambda: [torch.tensor([[4, 2, 6]])])

state.step(log_stats=True)

avg_tokens, max_tokens, balancedness = logger.info.call_args.args[3:6]
assert (avg_tokens, max_tokens, balancedness) == (6.0, 8.0, 0.75)
assert model_state.expert_load_window[0].tolist() == [[4, 2, 6]]
33 changes: 13 additions & 20 deletions tests/kernels/moe/test_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -768,13 +768,12 @@ def test_custom(

@pytest.mark.parametrize("record_enabled", [True, False])
@pytest.mark.parametrize(
"l2p_map, replica_count, num_physical, topk_ids, expected_out, expected_load",
"l2p_map, replica_count, topk_ids, expected_out, expected_load",
[
pytest.param(
# logical i → physical i
[[0], [1], [2], [3]],
[1, 1, 1, 1],
4,
[[0, 1], [2, 3], [0, 2]],
[[0, 1], [2, 3], [0, 2]],
[2, 1, 2, 1],
Expand All @@ -784,20 +783,18 @@ def test_custom(
# logical 0→3, 1→0, 2→1, 3→2
[[3], [0], [1], [2]],
[1, 1, 1, 1],
4,
[[0, 1], [2, 3], [0, 2]],
[[3, 0], [1, 2], [3, 1]],
[1, 2, 1, 2],
[2, 1, 2, 1],
id="shuffled",
),
pytest.param(
# logical 0→5, 1→2, 2→7, 3→0 in a larger physical space
[[5], [2], [7], [0]],
[1, 1, 1, 1],
8,
[[0, 1], [2, 3]],
[[5, 2], [7, 0]],
[1, 0, 1, 0, 0, 1, 0, 1],
[1, 1, 1, 1],
id="sparse",
),
],
Expand All @@ -806,14 +803,13 @@ def test_eplb_map_no_redundancy(
record_enabled,
l2p_map,
replica_count,
num_physical,
topk_ids,
expected_out,
expected_load,
):
l2p = torch.tensor(l2p_map, dtype=torch.int64, device=DEVICE)
rc = torch.tensor(replica_count, dtype=torch.int64, device=DEVICE)
load = torch.zeros(num_physical, dtype=torch.int32, device=DEVICE)
load = torch.zeros(len(l2p_map), dtype=torch.int32, device=DEVICE)
rec = torch.tensor(record_enabled, dtype=torch.bool, device=DEVICE)
ids = torch.tensor(topk_ids, dtype=torch.int32, device=DEVICE)

Expand Down Expand Up @@ -863,18 +859,19 @@ def test_eplb_map_hot_expert_replica_balance(top_k, R):
)
topk_ids[:, 0] = 0

load = torch.zeros(num_physical, dtype=torch.int32, device=DEVICE)
load = torch.zeros(num_logical, dtype=torch.int32, device=DEVICE)
rec = torch.tensor(True, dtype=torch.bool, device=DEVICE)

eplb_map_to_physical_and_record(
out = eplb_map_to_physical_and_record(
topk_ids=topk_ids,
expert_load_view=load,
logical_to_physical_map=l2p,
logical_replica_count=rc,
record_enabled=rec,
)

hot_load = load[:R].float()
assert load[0].item() == num_tokens
hot_load = torch.bincount(out.flatten(), minlength=num_physical)[:R].float()
max_mean = (hot_load.max() / hot_load.mean()).item()
assert max_mean < 1.15, (
f"Hot expert replicas uneven: {hot_load.tolist()}, max/mean={max_mean:.3f}"
Expand All @@ -883,46 +880,43 @@ def test_eplb_map_hot_expert_replica_balance(top_k, R):

@pytest.mark.parametrize("record_enabled", [True, False])
@pytest.mark.parametrize(
"l2p_map, replica_count, num_physical, topk_ids, expected_out, expected_load",
"l2p_map, replica_count, topk_ids, expected_out, expected_load",
[
pytest.param(
# experts 0,1 have 2 replicas; 2,3 have 1
[[0, 4], [1, 5], [2, -1], [3, -1]],
[2, 2, 1, 1],
6,
[[0, 1], [2, 3], [0, 2]],
# replica = (token_idx * KNUTH) & 0xFFFFFFFF % R.
# token 0 hash=0x00000000: %2=0, %1=0.
# token 1 hash=0x9E3779B9: %2=1, %1=0.
# token 2 hash=0x3C6EF372: %2=0, %1=0.
[[0, 1], [2, 3], [0, 2]],
[2, 1, 2, 1, 0, 0],
[2, 1, 2, 1],
id="partial",
),
pytest.param(
# all 4 experts have 2 replicas
[[0, 4], [1, 5], [2, 6], [3, 7]],
[2, 2, 2, 2],
8,
[[0, 1], [2, 3], [0, 2]],
# token 0 hash=0x00000000: %2=0.
# token 1 hash=0x9E3779B9: %2=1.
# token 2 hash=0x3C6EF372: %2=0.
[[0, 1], [6, 7], [0, 2]],
[2, 1, 1, 0, 0, 0, 1, 1],
[2, 1, 2, 1],
id="full",
),
pytest.param(
# expert 0: 4 replicas, experts 1,2: 2 replicas
[[0, 3, 5, 7], [1, 4, -1, -1], [2, 6, -1, -1]],
[4, 2, 2],
8,
[[0, 1], [2, 0], [1, 2]],
# token 0 hash=0x00000000: %4=0, %2=0.
# token 1 hash=0x9E3779B9: %4=1, %2=1.
# token 2 hash=0x3C6EF372: %4=2, %2=0.
[[0, 1], [6, 3], [1, 2]],
[1, 2, 1, 1, 0, 0, 1, 0],
[2, 2, 2],
id="uneven",
),
],
Expand All @@ -931,14 +925,13 @@ def test_eplb_map_with_redundancy(
record_enabled,
l2p_map,
replica_count,
num_physical,
topk_ids,
expected_out,
expected_load,
):
l2p = torch.tensor(l2p_map, dtype=torch.int64, device=DEVICE)
rc = torch.tensor(replica_count, dtype=torch.int64, device=DEVICE)
load = torch.zeros(num_physical, dtype=torch.int32, device=DEVICE)
load = torch.zeros(len(l2p_map), dtype=torch.int32, device=DEVICE)
rec = torch.tensor(record_enabled, dtype=torch.bool, device=DEVICE)
ids = torch.tensor(topk_ids, dtype=torch.int32, device=DEVICE)

Expand Down
4 changes: 2 additions & 2 deletions vllm/distributed/elastic_ep/elastic_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,7 @@ def switch_and_prepare(self) -> tuple[GroupCoordinator | None, ...]:
model.expert_weights = []
with set_current_vllm_config(self.worker.vllm_config):
model.set_eplb_state(
eplb_model_state.expert_load_pass_buffer,
eplb_model_state.expert_load_pass,
eplb_model_state.logical_to_physical_map,
eplb_model_state.logical_replica_count,
)
Expand Down Expand Up @@ -575,7 +575,7 @@ def _perform_eplb_reshuffle(
is_async_enabled = eplb_state.is_async
run_async = async_op and is_async_enabled
eplb_state.is_async = run_async
eplb_state.rearrange(rank_mapping=rank_mapping, use_last_expert_load=True)
eplb_state.rearrange(rank_mapping=rank_mapping)
if not run_async:
# Wait for non-blocking expert resharding copies before continuing
# the Elastic EP reconfiguration.
Expand Down
Loading
Loading