Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
5dc4639
init
SageMoore Mar 17, 2026
44f4f11
not working
SageMoore Mar 17, 2026
ace8304
init
SageMoore Mar 19, 2026
094126b
cleanup
SageMoore Mar 19, 2026
e7e70de
remove queue
SageMoore Mar 19, 2026
f4fe482
comments
SageMoore Mar 19, 2026
4e5bf9a
more comments
SageMoore Mar 19, 2026
4b3ac69
fix for event sync
SageMoore Mar 23, 2026
95a0c0c
add EventWrapper
SageMoore Mar 23, 2026
2f7ed7f
minor refactor
SageMoore Mar 23, 2026
914cf4e
swap over to threading event
SageMoore Mar 23, 2026
13ac084
rename EventWrapper
SageMoore Mar 23, 2026
10845df
fix race
SageMoore Mar 23, 2026
2e37b21
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Mar 23, 2026
cf7c9c1
misc refactor
SageMoore Mar 23, 2026
d5177e4
add asserts
SageMoore Mar 23, 2026
837f665
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Mar 25, 2026
9a78b8e
more sync cleanups
SageMoore Mar 25, 2026
544c9ff
more cleanups
SageMoore Mar 26, 2026
bbde306
cleanup
SageMoore Mar 27, 2026
2e2dd8f
minor comment update
SageMoore Mar 27, 2026
dff2428
remove window_ready_event
SageMoore Mar 27, 2026
81bb2dd
cleanup
SageMoore Mar 27, 2026
67aadc5
AsyncEplbLayerResult rename
SageMoore Mar 27, 2026
3f6b4ce
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Mar 30, 2026
b526659
comments
SageMoore Mar 30, 2026
1cd8948
comments
SageMoore Mar 30, 2026
cea868a
bugfix
SageMoore Mar 30, 2026
26751de
bugfix
SageMoore Mar 31, 2026
d00793e
add event unit test
SageMoore Mar 31, 2026
0610ad4
add async worker unit tests
SageMoore Mar 31, 2026
5d0f02e
move move_to_workspace out of EplbState
SageMoore Mar 31, 2026
bf7b154
more unit tests
SageMoore Mar 31, 2026
a12732d
bugfix
SageMoore Mar 31, 2026
2d69c8f
misc bugfixes
SageMoore Apr 2, 2026
e812dd8
comment update
SageMoore Apr 2, 2026
d61149c
comment update
SageMoore Apr 2, 2026
4a17c8f
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Apr 2, 2026
0b64b6a
misc refactoring
SageMoore Apr 2, 2026
3545c13
misc refactoring
SageMoore Apr 3, 2026
f2b7647
remove extra tests
SageMoore Apr 3, 2026
894a1ef
fix extra ars
SageMoore Apr 3, 2026
2f1c851
cleanup
SageMoore Apr 3, 2026
99bf9c3
comments
SageMoore Apr 6, 2026
f2d3bcf
add debug log
SageMoore Apr 7, 2026
ae5709b
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Apr 8, 2026
cd99b1c
comment updates
SageMoore Apr 9, 2026
7c38536
Merge branch 'main' into sage/eplb-async-result
SageMoore Apr 9, 2026
d5eca69
Merge branch 'main' of https://github.com/neuralmagic/vllm into sage/…
SageMoore Apr 16, 2026
0736515
review comments
SageMoore Apr 16, 2026
9993a24
Merge branch 'main' into sage/eplb-async-result
tlrmchlsmth Apr 17, 2026
5b106b6
Merge branch 'main' into sage/eplb-async-result
tlrmchlsmth Apr 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 68 additions & 81 deletions vllm/distributed/eplb/async_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger

from .rebalance_execute import transfer_layer
from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer

if TYPE_CHECKING:
from .eplb_state import EplbModelState, EplbState
Expand Down Expand Up @@ -60,12 +60,15 @@ def run_rebalance_experts(
model_state: "EplbModelState",
eplb_state: "EplbState",
physical_to_logical_map_cpu: torch.Tensor,
) -> None:
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Compute new expert mappings for one rebalancing cycle.

Returns: new_physical_to_logical_map
"""
Comment thread
SageMoore marked this conversation as resolved.
Resolved
assert model_state.eplb_stats is not None
eplb_stats = model_state.eplb_stats

# Wait for the main thread's all-reduce and clone to complete before
# accessing the global_expert_load_window tensor.
# Wait for global_expert_load_window tensor to be ready.
assert model_state.window_ready_event is not None
model_state.window_ready_event.wait()
model_state.window_ready_event = None
Expand All @@ -87,17 +90,20 @@ def run_rebalance_experts(
)
assert new_physical_to_logical_map.device == torch.device("cpu")

model_state.new_physical_to_logical_map = new_physical_to_logical_map

max_slots = model_state.logical_to_physical_map.shape[-1]
padded_logical = torch.nn.functional.pad(
new_logical_to_physical_map = torch.nn.functional.pad(
new_logical_to_physical_map,
(0, max(0, max_slots - new_logical_to_physical_map.shape[-1])),
value=-1,
).to(model_state.logical_to_physical_map.device)
new_replica = new_logical_replica_count.to(model_state.logical_replica_count.device)
model_state.new_logical_to_physical_map = padded_logical
model_state.new_logical_replica_count = new_replica
new_logical_replica_count = new_logical_replica_count.to(
model_state.logical_replica_count.device
)
return (
new_physical_to_logical_map,
new_logical_to_physical_map,
new_logical_replica_count,
)


async def transfer_run_periodically(
Expand All @@ -112,77 +118,58 @@ async def transfer_run_periodically(

assert state.is_async
for model_state in state.model_states.values():
rebalancing_algorithm_executed = False
physical_to_logical_map_cpu = None
layer_idx = 0
current_num_layers = model_state.model.num_moe_layers
while (
model_state.rebalanced
and model_state.layer_to_transfer < current_num_layers
):
if not model_state.ep_buffer_ready and model_state.rebalanced:
# Polling the lock directly in the async thread avoids
# the thread switch overhead of asyncio.to_thread.
# This is typically faster than offloading to a worker thread.
while not model_state.buffer_lock.acquire(blocking=False):
await asyncio.sleep(0)
try:
if model_state.layer_to_transfer >= current_num_layers:
break
if (
not rebalancing_algorithm_executed
or model_state.new_physical_to_logical_map is None
):
# Move the physical_to_logical_map to CPU
# for rebalancing and transfer_layer.
physical_to_logical_map_cpu = (
model_state.physical_to_logical_map.cpu()
)
run_rebalance_experts(
model_state, state, physical_to_logical_map_cpu
)
rebalancing_algorithm_executed = True
logger.info(
"Async worker computed new indices for model %s",
model_state.model_name,
)

assert model_state.new_physical_to_logical_map is not None
assert physical_to_logical_map_cpu is not None

layer_idx = model_state.layer_to_transfer
old_layer_indices = physical_to_logical_map_cpu[layer_idx]
new_layer_indices = model_state.new_physical_to_logical_map[
layer_idx
]

# Wait for the main thread to finish consuming the buffer
# before initiating an EPLB transfer on another layer.
if model_state.buffer_consumed_event is not None:
cuda_stream.wait_event(model_state.buffer_consumed_event)
model_state.buffer_consumed_event = None

(
model_state.is_unchanged,
model_state.is_received_locally,
model_state.recv_metadata,
) = await transfer_layer(
old_layer_indices=old_layer_indices,
new_layer_indices=new_layer_indices,
expert_weights=model_state.model.expert_weights[layer_idx],
expert_weights_buffer=model_state.expert_buffer,
ep_group=eplb_group,
is_profile=is_profile,
cuda_stream=cuda_stream,
)
event = torch.cuda.Event(blocking=False)
cuda_stream.record_event(event)
model_state.buffer_ready_event = event
model_state.ep_buffer_ready = 1
finally:
model_state.buffer_lock.release()
else:
if not model_state.rebalanced:
break
await asyncio.sleep(0.001)

# Snapshot the physical_to_logical_map and copy it to CPU
Comment thread
SageMoore marked this conversation as resolved.
Resolved
with model_state.map_lock:
physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu()

(
new_physical_to_logical_map,
new_logical_to_physical_map,
new_logical_replica_count,
) = run_rebalance_experts(model_state, state, physical_to_logical_map_cpu)
logger.info(
"Async worker computed new indices for model %s",
model_state.model_name,
)

while model_state.rebalanced and layer_idx < current_num_layers:
Comment thread
SageMoore marked this conversation as resolved.
Resolved
(
is_unchanged,
is_received_locally,
recv_metadata,
) = await transfer_layer(
old_layer_indices=physical_to_logical_map_cpu[layer_idx],
new_layer_indices=new_physical_to_logical_map[layer_idx],
expert_weights=model_state.model.expert_weights[layer_idx],
expert_weights_buffer=model_state.expert_buffer,
ep_group=eplb_group,
is_profile=is_profile,
cuda_stream=cuda_stream,
)

# Wait until all writes to expert_buffer have finished before making the
# AsynEPLBLayerResult visible to the main thread.
cuda_stream.synchronize()

# This event guarantees that expert_buffer will not be overwritten by
# subsequent iterations of this loop until the main thread has consumed
# it. Record is called by the main thread after move_from_buffer().
consumed_event = torch.cuda.Event()
cuda_stream.wait_event(consumed_event)
Comment thread
SageMoore marked this conversation as resolved.
Resolved

model_state.pending_result = AsyncEPLBLayerResult(
layer_idx=layer_idx,
new_physical_to_logical_map=new_physical_to_logical_map,
new_logical_to_physical_map=new_logical_to_physical_map,
new_logical_replica_count=new_logical_replica_count,
is_unchanged=is_unchanged,
is_received_locally=is_received_locally,
recv_metadata=recv_metadata,
consumed_event=consumed_event,
)
Comment on lines +145 to +152

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SageMoore as you pointed out to me, pending_result is read by one thread and written by another, and this relies on the GIL for correctness.

IMO we should lock model_state.pending_result it to guard against bugs. Can be a separate PR, but in this one we should make sure to comment that we are relying on the GIL

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is also true for model_state.rebalanced. I've updated the comment there as well. We can address both of these in a follow on PR.

layer_idx += 1

state.rearrange_event.clear()
Loading