From 5dc46399b59726d8eeb35fb7639307e56a19f6f4 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 17 Mar 2026 14:03:29 +0000 Subject: [PATCH 01/43] init Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 167 ++++++++++-------- vllm/distributed/eplb/eplb_state.py | 196 +++++++-------------- vllm/distributed/eplb/rebalance_execute.py | 40 +++++ 3 files changed, 191 insertions(+), 212 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 7e753fdbf41e..5752e2be6470 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -14,7 +14,7 @@ from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger -from .rebalance_execute import transfer_layer +from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer if TYPE_CHECKING: from .eplb_state import EplbModelState, EplbState @@ -60,7 +60,14 @@ def run_rebalance_experts( model_state: "EplbModelState", eplb_state: "EplbState", physical_to_logical_map_cpu: torch.Tensor, -) -> None: +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute new expert mappings for one rebalancing cycle. + + Returns: + (new_physical_to_logical_map, new_logical_to_physical_map, + new_logical_replica_count) — tensors shared across all + AsyncEPLBLayerResult objects produced in this cycle. + """ assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats @@ -87,17 +94,20 @@ def run_rebalance_experts( ) assert new_physical_to_logical_map.device == torch.device("cpu") - model_state.new_physical_to_logical_map = new_physical_to_logical_map - max_slots = model_state.logical_to_physical_map.shape[-1] - padded_logical = torch.nn.functional.pad( + new_logical_to_physical_map = torch.nn.functional.pad( new_logical_to_physical_map, (0, max(0, max_slots - new_logical_to_physical_map.shape[-1])), value=-1, ).to(model_state.logical_to_physical_map.device) - new_replica = new_logical_replica_count.to(model_state.logical_replica_count.device) - model_state.new_logical_to_physical_map = padded_logical - model_state.new_logical_replica_count = new_replica + new_logical_replica_count = new_logical_replica_count.to( + model_state.logical_replica_count.device + ) + return ( + new_physical_to_logical_map, + new_logical_to_physical_map, + new_logical_replica_count, + ) async def transfer_run_periodically( @@ -112,77 +122,78 @@ async def transfer_run_periodically( assert state.is_async for model_state in state.model_states.values(): - rebalancing_algorithm_executed = False - physical_to_logical_map_cpu = None + new_eplb_maps: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None + physical_to_logical_map_cpu: torch.Tensor | None = None + layer_idx = 0 current_num_layers = model_state.model.num_moe_layers - while ( - model_state.rebalanced - and model_state.layer_to_transfer < current_num_layers - ): - if not model_state.ep_buffer_ready and model_state.rebalanced: - # Polling the lock directly in the async thread avoids - # the thread switch overhead of asyncio.to_thread. - # This is typically faster than offloading to a worker thread. - while not model_state.buffer_lock.acquire(blocking=False): - await asyncio.sleep(0) - try: - if model_state.layer_to_transfer >= current_num_layers: - break - if ( - not rebalancing_algorithm_executed - or model_state.new_physical_to_logical_map is None - ): - # Move the physical_to_logical_map to CPU - # for rebalancing and transfer_layer. - physical_to_logical_map_cpu = ( - model_state.physical_to_logical_map.cpu() - ) - run_rebalance_experts( - model_state, state, physical_to_logical_map_cpu - ) - rebalancing_algorithm_executed = True - logger.info( - "Async worker computed new indices for model %s", - model_state.model_name, - ) - - assert model_state.new_physical_to_logical_map is not None - assert physical_to_logical_map_cpu is not None - - layer_idx = model_state.layer_to_transfer - old_layer_indices = physical_to_logical_map_cpu[layer_idx] - new_layer_indices = model_state.new_physical_to_logical_map[ - layer_idx - ] - - # Wait for the main thread to finish consuming the buffer - # before initiating an EPLB transfer on another layer. - if model_state.buffer_consumed_event is not None: - cuda_stream.wait_event(model_state.buffer_consumed_event) - model_state.buffer_consumed_event = None - - ( - model_state.is_unchanged, - model_state.is_received_locally, - model_state.recv_metadata, - ) = await transfer_layer( - old_layer_indices=old_layer_indices, - new_layer_indices=new_layer_indices, - expert_weights=model_state.model.expert_weights[layer_idx], - expert_weights_buffer=model_state.expert_buffer, - ep_group=eplb_group, - is_profile=is_profile, - cuda_stream=cuda_stream, - ) - event = torch.cuda.Event(blocking=False) - cuda_stream.record_event(event) - model_state.buffer_ready_event = event - model_state.ep_buffer_ready = 1 - finally: - model_state.buffer_lock.release() - else: - if not model_state.rebalanced: + + while model_state.rebalanced and layer_idx < current_num_layers: + # Polling the lock directly in the async thread avoids + # the thread switch overhead of asyncio.to_thread. + # This is typically faster than offloading to a worker thread. + while not model_state.buffer_lock.acquire(blocking=False): + await asyncio.sleep(0) + try: + if not model_state.rebalanced or layer_idx >= current_num_layers: break - await asyncio.sleep(0.001) + + if new_eplb_maps is None: + # Compute new expert mappings once per cycle. + physical_to_logical_map_cpu = ( + model_state.physical_to_logical_map.cpu() + ) + new_eplb_maps = run_rebalance_experts( + model_state, state, physical_to_logical_map_cpu + ) + logger.info( + "Async worker computed new indices for model %s", + model_state.model_name, + ) + + assert physical_to_logical_map_cpu is not None + ( + new_physical_to_logical_map, + new_logical_to_physical_map, + new_logical_replica_count, + ) = new_eplb_maps + + # Insert a GPU wait for the main thread to finish consuming + # the previous layer's buffer before overwriting it. + if model_state.buffer_consumed_event is not None: + cuda_stream.wait_event(model_state.buffer_consumed_event) + model_state.buffer_consumed_event = None + + ( + is_unchanged, + is_received_locally, + recv_metadata, + ) = await transfer_layer( + old_layer_indices=physical_to_logical_map_cpu[layer_idx], + new_layer_indices=new_physical_to_logical_map[layer_idx], + expert_weights=model_state.model.expert_weights[layer_idx], + expert_weights_buffer=model_state.expert_buffer, + ep_group=eplb_group, + is_profile=is_profile, + cuda_stream=cuda_stream, + ) + # Block until all GPU writes to the intermediate buffer are complete + # so that step() on the main thread can read the buffer without a + # separate event. + cuda_stream.synchronize() + + model_state.result_queue.put( + AsyncEPLBLayerResult( + layer_idx=layer_idx, + new_physical_to_logical_map=new_physical_to_logical_map, + new_logical_to_physical_map=new_logical_to_physical_map, + new_logical_replica_count=new_logical_replica_count, + is_unchanged=is_unchanged, + is_received_locally=is_received_locally, + recv_metadata=recv_metadata, + ) + ) + layer_idx += 1 + finally: + model_state.buffer_lock.release() state.rearrange_event.clear() diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 863b29f6ff87..f56b779b5d04 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -26,9 +26,10 @@ physical experts. """ +import queue as _queue import threading from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field import numpy as np import torch @@ -48,6 +49,7 @@ from .async_worker import start_async_worker from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( + AsyncEPLBLayerResult, RecvMetadata, move_from_buffer, rearrange_expert_weights_inplace, @@ -176,11 +178,6 @@ class EplbModelState: """ The lock to protect the expert buffer. """ - buffer_ready_event: torch.cuda.Event | None - """ - CUDA event recorded when the async worker finishes filling the buffer. - The main thread waits on this before consuming the buffer. - """ buffer_consumed_event: torch.cuda.Event | None """ CUDA event recorded after the main thread finishes consuming the buffer. @@ -191,59 +188,25 @@ class EplbModelState: CUDA event recorded after all-reduce and clone on the main thread. The async worker waits on this before accessing global_expert_load_window. """ - ep_buffer_ready: int - """ - The flag indicates whether the expert buffer is ready for transfer. - 0 or 1. - """ - layer_to_transfer: int - """ - The layer index to transfer in async mode. - """ rebalanced: bool """ The flag indicates whether the experts rebalance have been computed. """ - pending_global_ready_check: bool - """ - Whether the async EPLB needs to poll peers for buffer readiness. - """ eplb_stats: EplbStats | None """ EPLB stats for the model. """ - is_unchanged: np.ndarray - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - The size is same as the num of physical experts in the current layer. - """ - is_received_locally: np.ndarray - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - The size is same as the num of physical experts in the current layer. - """ - recv_metadata: RecvMetadata - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - """ cuda_device_index: int | None """ CUDA device index for the async EPLB worker thread. """ - new_physical_to_logical_map: torch.Tensor | None = None - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - the size is same as physical_to_logical_map + result_queue: _queue.SimpleQueue = field(default_factory=_queue.SimpleQueue) """ - new_logical_to_physical_map: torch.Tensor | None = None - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - the size is same as logical_to_physical_map - """ - new_logical_replica_count: torch.Tensor | None = None - """ - intermediate variable between `move_to_buffer` and `move_to_workspace`. - the size is same as logical_replica_count + Producer/consumer channel between the async worker and step(). + + The worker puts one AsyncEPLBLayerResult per layer after + cuda_stream.synchronize(), guaranteeing all GPU writes to expert_buffer + are complete. step() calls get_nowait() to check for a ready result. """ @@ -490,26 +453,11 @@ def add_model( model=model, expert_buffer=expert_buffer, buffer_lock=threading.Lock(), - buffer_ready_event=None, buffer_consumed_event=None, window_ready_event=None, - ep_buffer_ready=0, - layer_to_transfer=0, rebalanced=False, - pending_global_ready_check=False, eplb_stats=None, - is_unchanged=np.array([]), - is_received_locally=np.array([]), - recv_metadata=RecvMetadata( - recv_primary_mask=np.array([]), - recv_count=0, - recv_expert_ids=np.array([]), - recv_dst_rows=np.array([]), - ), cuda_device_index=self.cuda_device_index, - new_physical_to_logical_map=None, - new_logical_to_physical_map=None, - new_logical_replica_count=None, ) self.model_states[model_config.compute_hash()] = model_state self.num_valid_physical_experts = model.num_physical_experts @@ -618,14 +566,14 @@ def step( if self.is_async: for eplb_model_state in self.model_states.values(): - all_ranks_buffer_ready = False - if eplb_model_state.pending_global_ready_check: - all_ranks_buffer_ready = self._all_ranks_buffer_ready( - eplb_model_state - ) - if eplb_model_state.ep_buffer_ready and all_ranks_buffer_ready: + if ( + not eplb_model_state.result_queue.empty() + and self._all_ranks_result_ready(eplb_model_state) + ): + result = eplb_model_state.result_queue.get_nowait() self.move_to_workspace( model_state=eplb_model_state, + result=result, ep_group=ep_group, is_profile=is_profile, ) @@ -824,8 +772,6 @@ def rearrange( eplb_model_state.window_ready_event = sync_event eplb_model_state.rebalanced = True - eplb_model_state.layer_to_transfer = 0 - eplb_model_state.pending_global_ready_check = True # Signal async thread to start transferring layers if self.is_async and (not is_profile): self.rearrange_event.set() @@ -845,17 +791,11 @@ def start_async_loop( ) def _update_layer_mapping_from_new( - self, model_state: EplbModelState, layer: int + self, model_state: EplbModelState, result: AsyncEPLBLayerResult ) -> None: - if ( - model_state.new_physical_to_logical_map is None - or model_state.new_logical_to_physical_map is None - or model_state.new_logical_replica_count is None - ): - return - + layer = result.layer_idx target_device = model_state.physical_to_logical_map.device - new_physical = model_state.new_physical_to_logical_map + new_physical = result.new_physical_to_logical_map # If the number of physical experts has changed, then the new map needs to # be copied synchronously to avoid a race condition with the async worker if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: @@ -866,7 +806,7 @@ def _update_layer_mapping_from_new( ) logical_device = model_state.logical_to_physical_map.device - new_logical = model_state.new_logical_to_physical_map[layer].to(logical_device) + new_logical = result.new_logical_to_physical_map[layer].to(logical_device) max_slots = model_state.logical_to_physical_map.shape[-1] slot_delta = max_slots - new_logical.shape[-1] if slot_delta > 0: @@ -877,40 +817,46 @@ def _update_layer_mapping_from_new( replica_device = model_state.logical_replica_count.device model_state.logical_replica_count[layer].copy_( - model_state.new_logical_replica_count[layer].to(replica_device) + result.new_logical_replica_count[layer].to(replica_device) ) - def _all_ranks_buffer_ready(self, model_state: EplbModelState) -> bool: + def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: + """Return True only when every EP rank has a result queued. + + Prevents one rank from consuming its result (and updating + expert_weights) while another rank is still transferring, which would + cause a forward pass to run with partially-updated weights. + """ parallel_state = get_ep_group() + has_result = int(not model_state.result_queue.empty()) + cpu_group = getattr(parallel_state, "cpu_group", None) if cpu_group is not None and cpu_group.size() > 1: - flag = torch.tensor( - (int(model_state.ep_buffer_ready),), dtype=torch.int32, device="cpu" - ) + flag = torch.tensor((has_result,), dtype=torch.int32, device="cpu") all_reduce(flag, group=cpu_group) return int(flag.item()) == cpu_group.size() device_group = parallel_state.device_group if device_group.size() <= 1: - return bool(model_state.ep_buffer_ready) + return bool(has_result) device = getattr( parallel_state, "device", model_state.physical_to_logical_map.device ) - flag = torch.tensor( - (int(model_state.ep_buffer_ready),), dtype=torch.int32, device=device - ) + flag = torch.tensor((has_result,), dtype=torch.int32, device=device) all_reduce(flag, group=device_group) return int(flag.item()) == device_group.size() def move_to_workspace( self, model_state: EplbModelState, + result: AsyncEPLBLayerResult, ep_group: ProcessGroup, is_profile: bool = False, - ): - # We call move_to_workspace only when ep_buffer_ready is 1. - # It means we only need to wait for the lock for a short time. + ) -> None: + # The worker holds buffer_lock for the full duration of + # transfer_layer + synchronize, then releases it. We may need to wait + # briefly if the worker just finished and hasn't released yet. max_retries = 6 # 1 minute max retries = 0 while not model_state.buffer_lock.acquire(blocking=True, timeout=10.0): @@ -918,7 +864,7 @@ def move_to_workspace( if retries >= max_retries: raise RuntimeError( f"Rank {ep_group.rank()}: buffer_lock timeout after " - "{max_retries * 10}s" + f"{max_retries * 10}s" ) logger.warning( "Rank %d: EPLB buffer_lock acquire failed, retrying (%d/%d)", @@ -927,56 +873,37 @@ def move_to_workspace( max_retries, ) try: - assert model_state.new_physical_to_logical_map is not None - device_index = model_state.cuda_device_index or self.cuda_device_index - if model_state.buffer_ready_event is not None and device_index is not None: - stream = torch.cuda.current_stream(device=device_index) - stream.wait_event(model_state.buffer_ready_event) - model_state.buffer_ready_event = None - expert_weights = model_state.model.expert_weights[ - model_state.layer_to_transfer - ] - expert_weights_buffer = model_state.expert_buffer - new_indices = model_state.new_physical_to_logical_map[ - model_state.layer_to_transfer - ].numpy() move_from_buffer( - expert_weights=expert_weights, - expert_weights_buffers=expert_weights_buffer, - is_unchanged=model_state.is_unchanged, - is_received_locally=model_state.is_received_locally, - recv_metadata=model_state.recv_metadata, - new_indices=new_indices, + expert_weights=model_state.model.expert_weights[result.layer_idx], + expert_weights_buffers=model_state.expert_buffer, + is_unchanged=result.is_unchanged, + is_received_locally=result.is_received_locally, + recv_metadata=result.recv_metadata, + new_indices=result.new_physical_to_logical_map[ + result.layer_idx + ].numpy(), ep_rank=ep_group.rank(), ) - # Record event after consuming buffer to signal async thread - # that it's safe to overwrite the intermediate buffer + # Signal the async worker's GPU stream that the buffer is free to + # be overwritten for the next layer. consumed_event = torch.cuda.Event() consumed_event.record() model_state.buffer_consumed_event = consumed_event - transferred_layer = model_state.layer_to_transfer - self._update_layer_mapping_from_new(model_state, transferred_layer) - # After the main thread consumes, advance layer_to_transfer - model_state.layer_to_transfer += 1 - model_state.ep_buffer_ready = 0 + self._update_layer_mapping_from_new(model_state, result) logger.debug( "model %s successfully move_to_workspace layer %d", model_state.model_name, - transferred_layer, + result.layer_idx, ) - if model_state.layer_to_transfer >= model_state.model.num_moe_layers: - self.post_eplb(model_state, is_profile) + if result.layer_idx == model_state.model.num_moe_layers - 1: + self.post_eplb(model_state, result, is_profile) model_state.rebalanced = False - model_state.layer_to_transfer = 0 - model_state.pending_global_ready_check = False logger.info( - "finish async transfer for model %s rank %d layer %d", + "finish async transfer for model %s rank %d", model_state.model_name, ep_group.rank(), - model_state.model.num_moe_layers, ) - finally: try: model_state.buffer_lock.release() @@ -987,14 +914,15 @@ def move_to_workspace( str(e), ) - def post_eplb(self, model_state: EplbModelState, is_profile: bool = False) -> None: - assert model_state.new_physical_to_logical_map is not None - assert model_state.new_logical_to_physical_map is not None - assert model_state.new_logical_replica_count is not None - - model_state.new_physical_to_logical_map = None - model_state.new_logical_to_physical_map = None - model_state.new_logical_replica_count = None + def post_eplb( + self, + model_state: EplbModelState, + result: AsyncEPLBLayerResult, + is_profile: bool = False, + ) -> None: + # Maps are owned by the result object and will be garbage-collected + # naturally; nothing to clear here. + pass def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]: """ diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 7823ce4a35e3..218b8e74578b 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -44,6 +44,46 @@ class RecvMetadata: MoveToBufferResult = tuple[np.ndarray, np.ndarray, RecvMetadata] +@dataclass +class AsyncEPLBLayerResult: + """ + The result of one completed async EPLB layer transfer. + + Produced by the async worker after transfer_layer() completes and + cuda_stream.synchronize() is called — meaning all GPU writes to + expert_buffer are finished before step() ever dequeues this object. + + Consumed by EplbState.step() via EplbModelState.result_queue. + """ + + layer_idx: int + """Index of the MoE layer that was transferred.""" + new_physical_to_logical_map: torch.Tensor + """ + New physical→logical mapping for all layers, on CPU. + Shape: (num_moe_layers, num_physical_experts) + All layer results in a single cycle share the same tensor object. + """ + new_logical_to_physical_map: torch.Tensor + """ + New logical→physical mapping for all layers, on GPU. + Shape: (num_moe_layers, num_logical_experts, max_slots) + All layer results in a single cycle share the same tensor object. + """ + new_logical_replica_count: torch.Tensor + """ + New replica count for all layers, on GPU. + Shape: (num_moe_layers, num_logical_experts) + All layer results in a single cycle share the same tensor object. + """ + is_unchanged: np.ndarray + """Per-physical-expert flag: weight was not moved during transfer.""" + is_received_locally: np.ndarray + """Per-physical-expert flag: weight was received on this rank.""" + recv_metadata: RecvMetadata + """Metadata describing what was received during transfer_layer.""" + + def get_ep_ranks_with_experts_batch( expert_ids: np.ndarray, num_local_experts: int, From 44f4f113f7d02c5da02c77fbbc92a0cb5bf7acd7 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 17 Mar 2026 19:55:46 +0000 Subject: [PATCH 02/43] not working Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5752e2be6470..5834a98b6483 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -177,8 +177,8 @@ async def transfer_run_periodically( cuda_stream=cuda_stream, ) # Block until all GPU writes to the intermediate buffer are complete - # so that step() on the main thread can read the buffer without a - # separate event. + # so that step() on the main thread can read from the intermediate + # buffer without a separate GPU event. cuda_stream.synchronize() model_state.result_queue.put( From ace8304c8eab06c620ef0492c067ef9402107349 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 19 Mar 2026 17:48:36 +0000 Subject: [PATCH 03/43] init Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 118 +++++++++----------- vllm/distributed/eplb/eplb_state.py | 119 ++++++++------------- vllm/distributed/eplb/rebalance_execute.py | 7 ++ 3 files changed, 99 insertions(+), 145 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5834a98b6483..5db5940a6703 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -122,78 +122,60 @@ async def transfer_run_periodically( assert state.is_async for model_state in state.model_states.values(): - new_eplb_maps: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None - physical_to_logical_map_cpu: torch.Tensor | None = None layer_idx = 0 current_num_layers = model_state.model.num_moe_layers + # Snapshot physical_to_logical_map under the map lock so we don't + # race with _update_layer_mapping_from_new on the main thread. + with model_state.map_lock: + physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() + ( + new_physical_to_logical_map, + new_logical_to_physical_map, + new_logical_replica_count, + ) = run_rebalance_experts(model_state, state, physical_to_logical_map_cpu) + logger.info( + "Async worker computed new indices for model %s", + model_state.model_name, + ) + while model_state.rebalanced and layer_idx < current_num_layers: - # Polling the lock directly in the async thread avoids - # the thread switch overhead of asyncio.to_thread. - # This is typically faster than offloading to a worker thread. - while not model_state.buffer_lock.acquire(blocking=False): - await asyncio.sleep(0) - try: - if not model_state.rebalanced or layer_idx >= current_num_layers: - break - - if new_eplb_maps is None: - # Compute new expert mappings once per cycle. - physical_to_logical_map_cpu = ( - model_state.physical_to_logical_map.cpu() - ) - new_eplb_maps = run_rebalance_experts( - model_state, state, physical_to_logical_map_cpu - ) - logger.info( - "Async worker computed new indices for model %s", - model_state.model_name, - ) - - assert physical_to_logical_map_cpu is not None - ( - new_physical_to_logical_map, - new_logical_to_physical_map, - new_logical_replica_count, - ) = new_eplb_maps - - # Insert a GPU wait for the main thread to finish consuming - # the previous layer's buffer before overwriting it. - if model_state.buffer_consumed_event is not None: - cuda_stream.wait_event(model_state.buffer_consumed_event) - model_state.buffer_consumed_event = None - - ( - is_unchanged, - is_received_locally, - recv_metadata, - ) = await transfer_layer( - old_layer_indices=physical_to_logical_map_cpu[layer_idx], - new_layer_indices=new_physical_to_logical_map[layer_idx], - expert_weights=model_state.model.expert_weights[layer_idx], - expert_weights_buffer=model_state.expert_buffer, - ep_group=eplb_group, - is_profile=is_profile, - cuda_stream=cuda_stream, - ) - # Block until all GPU writes to the intermediate buffer are complete - # so that step() on the main thread can read from the intermediate - # buffer without a separate GPU event. - cuda_stream.synchronize() - - model_state.result_queue.put( - AsyncEPLBLayerResult( - layer_idx=layer_idx, - new_physical_to_logical_map=new_physical_to_logical_map, - new_logical_to_physical_map=new_logical_to_physical_map, - new_logical_replica_count=new_logical_replica_count, - is_unchanged=is_unchanged, - is_received_locally=is_received_locally, - recv_metadata=recv_metadata, - ) + ( + is_unchanged, + is_received_locally, + recv_metadata, + ) = await transfer_layer( + old_layer_indices=physical_to_logical_map_cpu[layer_idx], + new_layer_indices=new_physical_to_logical_map[layer_idx], + expert_weights=model_state.model.expert_weights[layer_idx], + expert_weights_buffer=model_state.expert_buffer, + ep_group=eplb_group, + is_profile=is_profile, + cuda_stream=cuda_stream, + ) + # Block until all GPU writes to expert_buffer are done so the + # main thread can safely call move_from_buffer. + cuda_stream.synchronize() + + # Create the consumed_event before publishing the result. + # The main thread records it after move_from_buffer(); the + # cuda_stream wait below prevents the next transfer_layer from + # overwriting expert_buffer before that recording completes. + consumed_event = torch.cuda.Event() + cuda_stream.wait_event(consumed_event) + + model_state.result_queue.put( + AsyncEPLBLayerResult( + layer_idx=layer_idx, + new_physical_to_logical_map=new_physical_to_logical_map, + new_logical_to_physical_map=new_logical_to_physical_map, + new_logical_replica_count=new_logical_replica_count, + is_unchanged=is_unchanged, + is_received_locally=is_received_locally, + recv_metadata=recv_metadata, + consumed_event=consumed_event, ) - layer_idx += 1 - finally: - model_state.buffer_lock.release() + ) + layer_idx += 1 state.rearrange_event.clear() diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index f56b779b5d04..b5380894328d 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -31,7 +31,6 @@ from collections.abc import Sequence from dataclasses import dataclass, field -import numpy as np import torch from torch.distributed import ProcessGroup, all_reduce @@ -50,7 +49,6 @@ from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( AsyncEPLBLayerResult, - RecvMetadata, move_from_buffer, rearrange_expert_weights_inplace, ) @@ -174,14 +172,11 @@ class EplbModelState: """ The buffer to store the expert weights during transfer. """ - buffer_lock: threading.Lock + map_lock: threading.Lock """ - The lock to protect the expert buffer. - """ - buffer_consumed_event: torch.cuda.Event | None - """ - CUDA event recorded after the main thread finishes consuming the buffer. - The async worker waits on this before writing to the buffer again. + Protects physical_to_logical_map (and related maps) against concurrent + reads by the async worker (snapshot at cycle start) and writes by the + main thread (_update_layer_mapping_from_new). """ window_ready_event: torch.cuda.Event | None """ @@ -452,8 +447,7 @@ def add_model( model_name=model_config.model, model=model, expert_buffer=expert_buffer, - buffer_lock=threading.Lock(), - buffer_consumed_event=None, + map_lock=threading.Lock(), window_ready_event=None, rebalanced=False, eplb_stats=None, @@ -794,16 +788,18 @@ def _update_layer_mapping_from_new( self, model_state: EplbModelState, result: AsyncEPLBLayerResult ) -> None: layer = result.layer_idx - target_device = model_state.physical_to_logical_map.device new_physical = result.new_physical_to_logical_map - # If the number of physical experts has changed, then the new map needs to - # be copied synchronously to avoid a race condition with the async worker - if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: - model_state.physical_to_logical_map = new_physical.to(target_device) - else: - model_state.physical_to_logical_map[layer].copy_( - new_physical[layer].to(target_device, non_blocking=True) - ) + with model_state.map_lock: + target_device = model_state.physical_to_logical_map.device + # If the number of physical experts has changed, then the new map + # needs to be replaced atomically to avoid a race with the async + # worker's snapshot (which reads physical_to_logical_map.cpu()). + if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: + model_state.physical_to_logical_map = new_physical.to(target_device) + else: + model_state.physical_to_logical_map[layer].copy_( + new_physical[layer].to(target_device, non_blocking=True) + ) logical_device = model_state.logical_to_physical_map.device new_logical = result.new_logical_to_physical_map[layer].to(logical_device) @@ -854,65 +850,34 @@ def move_to_workspace( ep_group: ProcessGroup, is_profile: bool = False, ) -> None: - # The worker holds buffer_lock for the full duration of - # transfer_layer + synchronize, then releases it. We may need to wait - # briefly if the worker just finished and hasn't released yet. - max_retries = 6 # 1 minute max - retries = 0 - while not model_state.buffer_lock.acquire(blocking=True, timeout=10.0): - retries += 1 - if retries >= max_retries: - raise RuntimeError( - f"Rank {ep_group.rank()}: buffer_lock timeout after " - f"{max_retries * 10}s" - ) - logger.warning( - "Rank %d: EPLB buffer_lock acquire failed, retrying (%d/%d)", - ep_group.rank(), - retries, - max_retries, - ) - try: - move_from_buffer( - expert_weights=model_state.model.expert_weights[result.layer_idx], - expert_weights_buffers=model_state.expert_buffer, - is_unchanged=result.is_unchanged, - is_received_locally=result.is_received_locally, - recv_metadata=result.recv_metadata, - new_indices=result.new_physical_to_logical_map[ - result.layer_idx - ].numpy(), - ep_rank=ep_group.rank(), - ) - # Signal the async worker's GPU stream that the buffer is free to - # be overwritten for the next layer. - consumed_event = torch.cuda.Event() - consumed_event.record() - model_state.buffer_consumed_event = consumed_event - - self._update_layer_mapping_from_new(model_state, result) - logger.debug( - "model %s successfully move_to_workspace layer %d", + move_from_buffer( + expert_weights=model_state.model.expert_weights[result.layer_idx], + expert_weights_buffers=model_state.expert_buffer, + is_unchanged=result.is_unchanged, + is_received_locally=result.is_received_locally, + recv_metadata=result.recv_metadata, + new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), + ep_rank=ep_group.rank(), + ) + # Unblock the async worker's cuda_stream: it queued a wait on this + # event before publishing the result, so recording it here signals + # that expert_buffer is free to be overwritten for the next layer. + result.consumed_event.record() + + self._update_layer_mapping_from_new(model_state, result) + logger.debug( + "model %s successfully move_to_workspace layer %d", + model_state.model_name, + result.layer_idx, + ) + if result.layer_idx == model_state.model.num_moe_layers - 1: + self.post_eplb(model_state, result, is_profile) + model_state.rebalanced = False + logger.info( + "finish async transfer for model %s rank %d", model_state.model_name, - result.layer_idx, + ep_group.rank(), ) - if result.layer_idx == model_state.model.num_moe_layers - 1: - self.post_eplb(model_state, result, is_profile) - model_state.rebalanced = False - logger.info( - "finish async transfer for model %s rank %d", - model_state.model_name, - ep_group.rank(), - ) - finally: - try: - model_state.buffer_lock.release() - except Exception as e: - logger.error( - "Rank %d: buffer_lock release failed in move_to_workspace: %s", - ep_group.rank(), - str(e), - ) def post_eplb( self, diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 218b8e74578b..1826aeebb0e3 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -82,6 +82,13 @@ class AsyncEPLBLayerResult: """Per-physical-expert flag: weight was received on this rank.""" recv_metadata: RecvMetadata """Metadata describing what was received during transfer_layer.""" + consumed_event: torch.cuda.Event + """ + Unrecorded CUDA event created by the async worker before publishing this + result. The async worker's cuda_stream waits on it, so the main thread + must call consumed_event.record() after move_from_buffer() completes to + allow the next layer's transfer to begin. + """ def get_ep_ranks_with_experts_batch( From 094126b1882e5f3ee918904cc45d62334d39d08d Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 19 Mar 2026 18:29:39 +0000 Subject: [PATCH 04/43] cleanup Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_state.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index b5380894328d..9fcd7c3b30d6 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -569,7 +569,6 @@ def step( model_state=eplb_model_state, result=result, ep_group=ep_group, - is_profile=is_profile, ) if self.expert_rearrangement_step >= self.expert_rearrangement_step_interval: @@ -848,7 +847,6 @@ def move_to_workspace( model_state: EplbModelState, result: AsyncEPLBLayerResult, ep_group: ProcessGroup, - is_profile: bool = False, ) -> None: move_from_buffer( expert_weights=model_state.model.expert_weights[result.layer_idx], @@ -871,7 +869,6 @@ def move_to_workspace( result.layer_idx, ) if result.layer_idx == model_state.model.num_moe_layers - 1: - self.post_eplb(model_state, result, is_profile) model_state.rebalanced = False logger.info( "finish async transfer for model %s rank %d", @@ -879,16 +876,6 @@ def move_to_workspace( ep_group.rank(), ) - def post_eplb( - self, - model_state: EplbModelState, - result: AsyncEPLBLayerResult, - is_profile: bool = False, - ) -> None: - # Maps are owned by the result object and will be garbage-collected - # naturally; nothing to clear here. - pass - def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]: """ All-reduce a list of tensors. From e7e70de49cedb9b5cba017af1b7eba6fd1f4ee37 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 19 Mar 2026 18:42:06 +0000 Subject: [PATCH 05/43] remove queue Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 20 +++++++++----------- vllm/distributed/eplb/eplb_state.py | 22 +++++++++++----------- vllm/distributed/eplb/rebalance_execute.py | 2 +- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5db5940a6703..bbc65958e8f4 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -164,17 +164,15 @@ async def transfer_run_periodically( consumed_event = torch.cuda.Event() cuda_stream.wait_event(consumed_event) - model_state.result_queue.put( - AsyncEPLBLayerResult( - layer_idx=layer_idx, - new_physical_to_logical_map=new_physical_to_logical_map, - new_logical_to_physical_map=new_logical_to_physical_map, - new_logical_replica_count=new_logical_replica_count, - is_unchanged=is_unchanged, - is_received_locally=is_received_locally, - recv_metadata=recv_metadata, - consumed_event=consumed_event, - ) + model_state.pending_result = AsyncEPLBLayerResult( + layer_idx=layer_idx, + new_physical_to_logical_map=new_physical_to_logical_map, + new_logical_to_physical_map=new_logical_to_physical_map, + new_logical_replica_count=new_logical_replica_count, + is_unchanged=is_unchanged, + is_received_locally=is_received_locally, + recv_metadata=recv_metadata, + consumed_event=consumed_event, ) layer_idx += 1 diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 9fcd7c3b30d6..4e92e7c7c935 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -26,10 +26,9 @@ physical experts. """ -import queue as _queue import threading from collections.abc import Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass import torch from torch.distributed import ProcessGroup, all_reduce @@ -195,13 +194,13 @@ class EplbModelState: """ CUDA device index for the async EPLB worker thread. """ - result_queue: _queue.SimpleQueue = field(default_factory=_queue.SimpleQueue) + pending_result: AsyncEPLBLayerResult | None = None """ - Producer/consumer channel between the async worker and step(). - - The worker puts one AsyncEPLBLayerResult per layer after - cuda_stream.synchronize(), guaranteeing all GPU writes to expert_buffer - are complete. step() calls get_nowait() to check for a ready result. + Set by the async worker after cuda_stream.synchronize() completes for a + layer, guaranteeing all GPU writes to expert_buffer are done. Consumed + and reset to None by step() via move_to_workspace(). At most one result + is ever pending at a time — the GPU ordering via consumed_event ensures + the previous result is consumed before the next one is produced. """ @@ -561,10 +560,11 @@ def step( if self.is_async: for eplb_model_state in self.model_states.values(): if ( - not eplb_model_state.result_queue.empty() + eplb_model_state.pending_result is not None and self._all_ranks_result_ready(eplb_model_state) ): - result = eplb_model_state.result_queue.get_nowait() + result = eplb_model_state.pending_result + eplb_model_state.pending_result = None self.move_to_workspace( model_state=eplb_model_state, result=result, @@ -823,7 +823,7 @@ def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: cause a forward pass to run with partially-updated weights. """ parallel_state = get_ep_group() - has_result = int(not model_state.result_queue.empty()) + has_result = int(model_state.pending_result is not None) cpu_group = getattr(parallel_state, "cpu_group", None) if cpu_group is not None and cpu_group.size() > 1: diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 1826aeebb0e3..7a0d297fbeb5 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -53,7 +53,7 @@ class AsyncEPLBLayerResult: cuda_stream.synchronize() is called — meaning all GPU writes to expert_buffer are finished before step() ever dequeues this object. - Consumed by EplbState.step() via EplbModelState.result_queue. + Consumed by EplbState.step() via EplbModelState.pending_result. """ layer_idx: int From f4fe48247f550354154dc8747a715b001eac368d Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 19 Mar 2026 20:47:21 +0000 Subject: [PATCH 06/43] comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 24 ++++++++++------------- vllm/distributed/eplb/eplb_state.py | 28 ++++++++------------------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index bbc65958e8f4..f821fdcfafa9 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -63,16 +63,12 @@ def run_rebalance_experts( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Compute new expert mappings for one rebalancing cycle. - Returns: - (new_physical_to_logical_map, new_logical_to_physical_map, - new_logical_replica_count) — tensors shared across all - AsyncEPLBLayerResult objects produced in this cycle. + Returns: new_physical_to_logical_map """ assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats - # Wait for the main thread's all-reduce and clone to complete before - # accessing the global_expert_load_window tensor. + # Wait for global_expert_load_window tensor to be ready. assert model_state.window_ready_event is not None model_state.window_ready_event.wait() model_state.window_ready_event = None @@ -125,10 +121,10 @@ async def transfer_run_periodically( layer_idx = 0 current_num_layers = model_state.model.num_moe_layers - # Snapshot physical_to_logical_map under the map lock so we don't - # race with _update_layer_mapping_from_new on the main thread. + # Snapshot the physical_to_logical_map and copy it to CPU with model_state.map_lock: physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() + ( new_physical_to_logical_map, new_logical_to_physical_map, @@ -153,14 +149,14 @@ async def transfer_run_periodically( is_profile=is_profile, cuda_stream=cuda_stream, ) - # Block until all GPU writes to expert_buffer are done so the - # main thread can safely call move_from_buffer. + + # Wait until all writes to expert_buffer have finished before making the + # AsynEPLBLayerResult visible to the main thread. cuda_stream.synchronize() - # Create the consumed_event before publishing the result. - # The main thread records it after move_from_buffer(); the - # cuda_stream wait below prevents the next transfer_layer from - # overwriting expert_buffer before that recording completes. + # This event guarantees that expert_buffer will not be overwritten by + # subsequent iterations of this loop until the main thread has consumed + # it. Record is called by the main thread after move_from_buffer(). consumed_event = torch.cuda.Event() cuda_stream.wait_event(consumed_event) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 4e92e7c7c935..d0e671efc83e 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -173,9 +173,8 @@ class EplbModelState: """ map_lock: threading.Lock """ - Protects physical_to_logical_map (and related maps) against concurrent - reads by the async worker (snapshot at cycle start) and writes by the - main thread (_update_layer_mapping_from_new). + Protects physical_to_logical_map. Acquired by the async worker when copying the map + to CPU and by the main thread when updating the map with the result from rearrange. """ window_ready_event: torch.cuda.Event | None """ @@ -196,11 +195,9 @@ class EplbModelState: """ pending_result: AsyncEPLBLayerResult | None = None """ - Set by the async worker after cuda_stream.synchronize() completes for a - layer, guaranteeing all GPU writes to expert_buffer are done. Consumed - and reset to None by step() via move_to_workspace(). At most one result - is ever pending at a time — the GPU ordering via consumed_event ensures - the previous result is consumed before the next one is produced. + Set by the async worker after all writes to expert_buffer are done. Consumed + and reset to None by the main thread in move_to_workspace() after the contents of + expert_buffer have been transferred out. At most one result is pending at a time. """ @@ -790,9 +787,8 @@ def _update_layer_mapping_from_new( new_physical = result.new_physical_to_logical_map with model_state.map_lock: target_device = model_state.physical_to_logical_map.device - # If the number of physical experts has changed, then the new map - # needs to be replaced atomically to avoid a race with the async - # worker's snapshot (which reads physical_to_logical_map.cpu()). + # If the number of physical experts has changed, then the new map needs to + # be copied synchronously to avoid a race condition with the async worker if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: model_state.physical_to_logical_map = new_physical.to(target_device) else: @@ -816,12 +812,6 @@ def _update_layer_mapping_from_new( ) def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: - """Return True only when every EP rank has a result queued. - - Prevents one rank from consuming its result (and updating - expert_weights) while another rank is still transferring, which would - cause a forward pass to run with partially-updated weights. - """ parallel_state = get_ep_group() has_result = int(model_state.pending_result is not None) @@ -857,9 +847,7 @@ def move_to_workspace( new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), ep_rank=ep_group.rank(), ) - # Unblock the async worker's cuda_stream: it queued a wait on this - # event before publishing the result, so recording it here signals - # that expert_buffer is free to be overwritten for the next layer. + # Unblock the async worker result.consumed_event.record() self._update_layer_mapping_from_new(model_state, result) From 4e5bf9a28ecff12d7d04515c96151958ba2d4433 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 19 Mar 2026 20:52:01 +0000 Subject: [PATCH 07/43] more comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/rebalance_execute.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 7a0d297fbeb5..6f144e87e30c 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -48,12 +48,6 @@ class RecvMetadata: class AsyncEPLBLayerResult: """ The result of one completed async EPLB layer transfer. - - Produced by the async worker after transfer_layer() completes and - cuda_stream.synchronize() is called — meaning all GPU writes to - expert_buffer are finished before step() ever dequeues this object. - - Consumed by EplbState.step() via EplbModelState.pending_result. """ layer_idx: int @@ -62,19 +56,16 @@ class AsyncEPLBLayerResult: """ New physical→logical mapping for all layers, on CPU. Shape: (num_moe_layers, num_physical_experts) - All layer results in a single cycle share the same tensor object. """ new_logical_to_physical_map: torch.Tensor """ New logical→physical mapping for all layers, on GPU. Shape: (num_moe_layers, num_logical_experts, max_slots) - All layer results in a single cycle share the same tensor object. """ new_logical_replica_count: torch.Tensor """ New replica count for all layers, on GPU. Shape: (num_moe_layers, num_logical_experts) - All layer results in a single cycle share the same tensor object. """ is_unchanged: np.ndarray """Per-physical-expert flag: weight was not moved during transfer.""" From 4b3ac6948366433bbce7885cb50aa0a62b9222e8 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 17:20:57 +0000 Subject: [PATCH 08/43] fix for event sync Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 5 ++++- vllm/distributed/eplb/eplb_state.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index f821fdcfafa9..5dfe05ea8536 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -158,7 +158,6 @@ async def transfer_run_periodically( # subsequent iterations of this loop until the main thread has consumed # it. Record is called by the main thread after move_from_buffer(). consumed_event = torch.cuda.Event() - cuda_stream.wait_event(consumed_event) model_state.pending_result = AsyncEPLBLayerResult( layer_idx=layer_idx, @@ -170,6 +169,10 @@ async def transfer_run_periodically( recv_metadata=recv_metadata, consumed_event=consumed_event, ) + + while model_state.pending_result is not None: + await asyncio.sleep(0) + cuda_stream.wait_event(consumed_event) layer_idx += 1 state.rearrange_event.clear() diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index d0e671efc83e..ae4472e6fe2a 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -560,11 +560,8 @@ def step( eplb_model_state.pending_result is not None and self._all_ranks_result_ready(eplb_model_state) ): - result = eplb_model_state.pending_result - eplb_model_state.pending_result = None self.move_to_workspace( model_state=eplb_model_state, - result=result, ep_group=ep_group, ) @@ -835,9 +832,10 @@ def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: def move_to_workspace( self, model_state: EplbModelState, - result: AsyncEPLBLayerResult, ep_group: ProcessGroup, ) -> None: + result = model_state.pending_result + assert result is not None move_from_buffer( expert_weights=model_state.model.expert_weights[result.layer_idx], expert_weights_buffers=model_state.expert_buffer, @@ -847,8 +845,10 @@ def move_to_workspace( new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), ep_rank=ep_group.rank(), ) + # Unblock the async worker result.consumed_event.record() + model_state.pending_result = None self._update_layer_mapping_from_new(model_state, result) logger.debug( From 95a0c0cb81c9470a1d06ee48db310a2b268beb3d Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 20:23:03 +0000 Subject: [PATCH 09/43] add EventWrapper Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 7 ++--- vllm/distributed/eplb/eplb_utils.py | 35 ++++++++++++++++++++++ vllm/distributed/eplb/rebalance_execute.py | 10 +++---- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5dfe05ea8536..707c655904c5 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -14,6 +14,7 @@ from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger +from .eplb_utils import EventWrapper from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer if TYPE_CHECKING: @@ -157,7 +158,7 @@ async def transfer_run_periodically( # This event guarantees that expert_buffer will not be overwritten by # subsequent iterations of this loop until the main thread has consumed # it. Record is called by the main thread after move_from_buffer(). - consumed_event = torch.cuda.Event() + consumed_event = EventWrapper() model_state.pending_result = AsyncEPLBLayerResult( layer_idx=layer_idx, @@ -170,9 +171,7 @@ async def transfer_run_periodically( consumed_event=consumed_event, ) - while model_state.pending_result is not None: - await asyncio.sleep(0) - cuda_stream.wait_event(consumed_event) + consumed_event.wait(cuda_stream) layer_idx += 1 state.rearrange_event.clear() diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 455848341a47..96d05f07569e 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -3,6 +3,9 @@ """Utility functions for EPLB (Expert Parallel Load Balancing).""" import os +import time + +import torch from vllm.config import ParallelConfig from vllm.logger import init_logger @@ -10,6 +13,38 @@ logger = init_logger(__name__) +class EventWrapper: + """ + A CUDA event with a CPU-side flag that guards against calling + event.wait() before event.record() has been called (which would + be a no-op and break ordering guarantees). + + Typical usage across two threads: + + Producer thread (async worker): + event.wait() # spins until consumer calls record(), then inserts + # GPU-side dependency on the current CUDA stream + + Consumer thread (main thread): + # ... consume the buffer ... + event.record() # records on the current CUDA stream, then sets flag + """ + + def __init__(self): + self.event = torch.cuda.Event() + self.record_call = False + + def wait(self, stream: torch.cuda.Stream | None = None): + while self.record_call is False: + time.sleep(0) + self.event.wait(stream) + self.record_call = False + + def record(self, stream: torch.cuda.Stream | None = None): + self.event.record(stream) + self.record_call = True + + def override_envs_for_eplb(parallel_config: ParallelConfig) -> None: """ Override environment variables for EPLB when specific conditions are met. diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 6f144e87e30c..f50a5821a82f 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -19,6 +19,7 @@ get_global_rank, ) +from vllm.distributed.eplb.eplb_utils import EventWrapper from vllm.distributed.parallel_state import get_ep_group from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger @@ -73,12 +74,11 @@ class AsyncEPLBLayerResult: """Per-physical-expert flag: weight was received on this rank.""" recv_metadata: RecvMetadata """Metadata describing what was received during transfer_layer.""" - consumed_event: torch.cuda.Event + consumed_event: EventWrapper """ - Unrecorded CUDA event created by the async worker before publishing this - result. The async worker's cuda_stream waits on it, so the main thread - must call consumed_event.record() after move_from_buffer() completes to - allow the next layer's transfer to begin. + Created by the async worker before publishing this result. The main thread + calls consumed_event.record() after move_from_buffer() completes, which + unblocks the async worker's consumed_event.wait() call. """ From 2f7ed7f2334a728d37166104afd0a14c34883401 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 20:24:42 +0000 Subject: [PATCH 10/43] minor refactor Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 707c655904c5..0f26fedffe78 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -171,7 +171,7 @@ async def transfer_run_periodically( consumed_event=consumed_event, ) - consumed_event.wait(cuda_stream) + consumed_event.wait(stream=cuda_stream) layer_idx += 1 state.rearrange_event.clear() From 914cf4eb313ac66c72c4d4cda8dbab371881cdb7 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 20:29:35 +0000 Subject: [PATCH 11/43] swap over to threading event Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_utils.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 96d05f07569e..ba625917bfda 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -3,7 +3,7 @@ """Utility functions for EPLB (Expert Parallel Load Balancing).""" import os -import time +import threading import torch @@ -31,18 +31,17 @@ class EventWrapper: """ def __init__(self): - self.event = torch.cuda.Event() - self.record_call = False + self._event = torch.cuda.Event() + self._recorded = threading.Event() def wait(self, stream: torch.cuda.Stream | None = None): - while self.record_call is False: - time.sleep(0) - self.event.wait(stream) - self.record_call = False + self._recorded.wait() + self._event.wait(stream) + self._recorded.clear() def record(self, stream: torch.cuda.Stream | None = None): - self.event.record(stream) - self.record_call = True + self._event.record(stream) + self._recorded.set() def override_envs_for_eplb(parallel_config: ParallelConfig) -> None: From 13ac08480182fdb1034359df9c18963e858beffb Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 20:30:28 +0000 Subject: [PATCH 12/43] rename EventWrapper Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 4 ++-- vllm/distributed/eplb/eplb_utils.py | 2 +- vllm/distributed/eplb/rebalance_execute.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 0f26fedffe78..7c6aa49c5ff9 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -14,7 +14,7 @@ from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger -from .eplb_utils import EventWrapper +from .eplb_utils import EPLBEvent from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer if TYPE_CHECKING: @@ -158,7 +158,7 @@ async def transfer_run_periodically( # This event guarantees that expert_buffer will not be overwritten by # subsequent iterations of this loop until the main thread has consumed # it. Record is called by the main thread after move_from_buffer(). - consumed_event = EventWrapper() + consumed_event = EPLBEvent() model_state.pending_result = AsyncEPLBLayerResult( layer_idx=layer_idx, diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index ba625917bfda..711650e6f432 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -13,7 +13,7 @@ logger = init_logger(__name__) -class EventWrapper: +class EPLBEvent: """ A CUDA event with a CPU-side flag that guards against calling event.wait() before event.record() has been called (which would diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index f50a5821a82f..163ecb20908c 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -19,7 +19,7 @@ get_global_rank, ) -from vllm.distributed.eplb.eplb_utils import EventWrapper +from vllm.distributed.eplb.eplb_utils import EPLBEvent from vllm.distributed.parallel_state import get_ep_group from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger @@ -74,7 +74,7 @@ class AsyncEPLBLayerResult: """Per-physical-expert flag: weight was received on this rank.""" recv_metadata: RecvMetadata """Metadata describing what was received during transfer_layer.""" - consumed_event: EventWrapper + consumed_event: EPLBEvent """ Created by the async worker before publishing this result. The main thread calls consumed_event.record() after move_from_buffer() completes, which From 10845df1d4e4016f8491970c579df633fc39899b Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 20:37:33 +0000 Subject: [PATCH 13/43] fix race Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index ae4472e6fe2a..a3b606b2de39 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -846,9 +846,9 @@ def move_to_workspace( ep_rank=ep_group.rank(), ) - # Unblock the async worker - result.consumed_event.record() + # Reset pending_result before ublocking the async worker model_state.pending_result = None + result.consumed_event.record() self._update_layer_mapping_from_new(model_state, result) logger.debug( From cf7c9c1dc5f65529a44b4120cbe7c88a913d88be Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 21:02:12 +0000 Subject: [PATCH 14/43] misc refactor Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 50e49728ae3d..35ad211baa57 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -62,14 +62,11 @@ def run_rebalance_experts( eplb_state: "EplbState", physical_to_logical_map_cpu: torch.Tensor, ) -> torch.Tensor: - """Compute new expert mappings for one rebalancing cycle. - - Returns: new_physical_to_logical_map - """ assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats - # Wait for global_expert_load_window tensor to be ready. + # Wait for the main thread's all-reduce and clone to complete before + # accessing the global_expert_load_window tensor. assert model_state.window_ready_event is not None model_state.window_ready_event.wait() model_state.window_ready_event = None From d5177e4f26cb9378beb81801c16245faa8b27991 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 23 Mar 2026 21:11:28 +0000 Subject: [PATCH 15/43] add asserts Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_utils.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 711650e6f432..64e2584c535b 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -14,22 +14,6 @@ class EPLBEvent: - """ - A CUDA event with a CPU-side flag that guards against calling - event.wait() before event.record() has been called (which would - be a no-op and break ordering guarantees). - - Typical usage across two threads: - - Producer thread (async worker): - event.wait() # spins until consumer calls record(), then inserts - # GPU-side dependency on the current CUDA stream - - Consumer thread (main thread): - # ... consume the buffer ... - event.record() # records on the current CUDA stream, then sets flag - """ - def __init__(self): self._event = torch.cuda.Event() self._recorded = threading.Event() @@ -40,6 +24,7 @@ def wait(self, stream: torch.cuda.Stream | None = None): self._recorded.clear() def record(self, stream: torch.cuda.Stream | None = None): + assert not self._recorded.is_set() self._event.record(stream) self._recorded.set() From 9a78b8e9e26c9c3190ef681f003049bfd9b67123 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Wed, 25 Mar 2026 21:04:30 +0000 Subject: [PATCH 16/43] more sync cleanups Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 12 +++++------ vllm/distributed/eplb/eplb_state.py | 30 +++++++++++---------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 35ad211baa57..71b687485168 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -61,6 +61,7 @@ def run_rebalance_experts( model_state: "EplbModelState", eplb_state: "EplbState", physical_to_logical_map_cpu: torch.Tensor, + cuda_stream: torch.cuda.Stream, ) -> torch.Tensor: assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats @@ -72,7 +73,8 @@ def run_rebalance_experts( model_state.window_ready_event = None # Move the global expert load window to CPU for computation. - global_expert_load_window = eplb_stats.global_expert_load_window.cpu() + with torch.cuda.stream(cuda_stream): + global_expert_load_window = eplb_stats.global_expert_load_window.cpu() # Compute new expert mappings for the model new_physical_to_logical_map = eplb_state.policy.rebalance_experts( global_expert_load_window, @@ -94,7 +96,7 @@ async def transfer_run_periodically( is_profile: bool = False, ) -> None: while True: - await asyncio.to_thread(state.rearrange_event.wait) + state.rearrange_event.wait() logger.info("async worker woke up for EPLB transfer") assert state.is_async @@ -103,11 +105,11 @@ async def transfer_run_periodically( current_num_layers = model_state.model.num_moe_layers # Snapshot the physical_to_logical_map and copy it to CPU - with model_state.map_lock: + with torch.cuda.stream(cuda_stream): physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() new_physical_to_logical_map = run_rebalance_experts( - model_state, state, physical_to_logical_map_cpu + model_state, state, physical_to_logical_map_cpu, cuda_stream ) logger.info( "Async worker computed new indices for model %s", @@ -149,5 +151,3 @@ async def transfer_run_periodically( consumed_event.wait(stream=cuda_stream) layer_idx += 1 - - state.rearrange_event.clear() diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index edfaa240e635..ce5c8cb83f12 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -45,6 +45,7 @@ from vllm.model_executor.models.interfaces import MixtureOfExperts from .async_worker import start_async_worker +from .eplb_utils import EPLBEvent from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( AsyncEPLBLayerResult, @@ -171,11 +172,6 @@ class EplbModelState: """ The buffer to store the expert weights during transfer. """ - map_lock: threading.Lock - """ - Protects physical_to_logical_map. Acquired by the async worker when copying the map - to CPU and by the main thread when updating the map with the result from rearrange. - """ window_ready_event: torch.cuda.Event | None """ CUDA event recorded after all-reduce and clone on the main thread. @@ -245,7 +241,7 @@ def __init__(self, parallel_config: ParallelConfig, device: torch.device): """ The flag indicates whether the EPLB is running in async mode. """ - self.rearrange_event = threading.Event() + self.rearrange_event: EPLBEvent = EPLBEvent() """ Event to signal when a new rearrangement is needed for the async thread. """ @@ -443,7 +439,6 @@ def add_model( model_name=model_config.model, model=model, expert_buffer=expert_buffer, - map_lock=threading.Lock(), window_ready_event=None, rebalanced=False, eplb_stats=None, @@ -764,7 +759,7 @@ def rearrange( eplb_model_state.rebalanced = True # Signal async thread to start transferring layers if self.is_async and (not is_profile): - self.rearrange_event.set() + self.rearrange_event.record() return None def start_async_loop( @@ -786,16 +781,15 @@ def _update_layer_mapping_from_new( layer = result.layer_idx new_physical = result.new_physical_to_logical_map - with model_state.map_lock: - target_device = model_state.physical_to_logical_map.device - # If the number of physical experts has changed, then the new map needs to - # be copied synchronously to avoid a race condition with the async worker - if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: - model_state.physical_to_logical_map = new_physical.to(target_device) - else: - model_state.physical_to_logical_map[layer].copy_( - new_physical[layer].to(target_device, non_blocking=True) - ) + target_device = model_state.physical_to_logical_map.device + # If the number of physical experts has changed, then the new map needs to + # be copied synchronously to avoid a race condition with the async worker + if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]: + model_state.physical_to_logical_map = new_physical.to(target_device) + else: + model_state.physical_to_logical_map[layer].copy_( + new_physical[layer].to(target_device, non_blocking=True) + ) num_logical_experts = model_state.logical_to_physical_map.shape[1] new_logical, new_replica_count = compute_logical_maps( From 544c9ff4c168ffdcaa3b7c975acb2a0f4b89e041 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 26 Mar 2026 18:19:55 +0000 Subject: [PATCH 17/43] more cleanups Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 11 +++++++++-- vllm/distributed/eplb/eplb_state.py | 4 ++-- vllm/distributed/eplb/eplb_utils.py | 2 +- vllm/distributed/eplb/rebalance_execute.py | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 71b687485168..dac8382a5421 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -14,7 +14,7 @@ from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger -from .eplb_utils import EPLBEvent +from .eplb_utils import EplbEvent from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer if TYPE_CHECKING: @@ -116,6 +116,10 @@ async def transfer_run_periodically( model_state.model_name, ) + # Execute one EPLB layer transfer per model forward pass. Each iteration + # of this loop will copy the new set of expert weights into + # model_state.expert_buffer, which will be consumed by the main thread in + # EplbState.step() while model_state.rebalanced and layer_idx < current_num_layers: ( is_unchanged, @@ -138,7 +142,7 @@ async def transfer_run_periodically( # This event guarantees that expert_buffer will not be overwritten by # subsequent iterations of this loop until the main thread has consumed # it. Record is called by the main thread after move_from_buffer(). - consumed_event = EPLBEvent() + consumed_event = EplbEvent() model_state.pending_result = AsyncEPLBLayerResult( layer_idx=layer_idx, @@ -149,5 +153,8 @@ async def transfer_run_periodically( consumed_event=consumed_event, ) + # Block the this thread until the main thread finishes copying + # model_state.expert_buffer into + # model_state.model.expert_weights[layer_idx] consumed_event.wait(stream=cuda_stream) layer_idx += 1 diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index ce5c8cb83f12..e66e05e47615 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -45,7 +45,7 @@ from vllm.model_executor.models.interfaces import MixtureOfExperts from .async_worker import start_async_worker -from .eplb_utils import EPLBEvent +from .eplb_utils import EplbEvent from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( AsyncEPLBLayerResult, @@ -241,7 +241,7 @@ def __init__(self, parallel_config: ParallelConfig, device: torch.device): """ The flag indicates whether the EPLB is running in async mode. """ - self.rearrange_event: EPLBEvent = EPLBEvent() + self.rearrange_event: EplbEvent = EplbEvent() """ Event to signal when a new rearrangement is needed for the async thread. """ diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 64e2584c535b..46ff202f6325 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -13,7 +13,7 @@ logger = init_logger(__name__) -class EPLBEvent: +class EplbEvent: def __init__(self): self._event = torch.cuda.Event() self._recorded = threading.Event() diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index d254608e896d..2a211bf1d58d 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -19,7 +19,7 @@ get_global_rank, ) -from vllm.distributed.eplb.eplb_utils import EPLBEvent +from vllm.distributed.eplb.eplb_utils import EplbEvent from vllm.distributed.parallel_state import get_ep_group from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger @@ -64,7 +64,7 @@ class AsyncEPLBLayerResult: """Per-physical-expert flag: weight was received on this rank.""" recv_metadata: RecvMetadata """Metadata describing what was received during transfer_layer.""" - consumed_event: EPLBEvent + consumed_event: EplbEvent """ Created by the async worker before publishing this result. The main thread calls consumed_event.record() after move_from_buffer() completes, which From bbde306f626327550ee33519eee01e5521db7273 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 27 Mar 2026 14:14:11 +0000 Subject: [PATCH 18/43] cleanup Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 1 + vllm/distributed/eplb/eplb_state.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index dac8382a5421..5cf3b0fe5ba3 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -157,4 +157,5 @@ async def transfer_run_periodically( # model_state.expert_buffer into # model_state.model.expert_weights[layer_idx] consumed_event.wait(stream=cuda_stream) + assert model_state.pending_result is None layer_idx += 1 diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index e66e05e47615..3371497199da 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -848,10 +848,6 @@ def move_to_workspace( ep_rank=ep_group.rank(), ) - # Reset pending_result before ublocking the async worker - model_state.pending_result = None - result.consumed_event.record() - self._update_layer_mapping_from_new(model_state, result) logger.debug( "model %s successfully move_to_workspace layer %d", @@ -866,6 +862,10 @@ def move_to_workspace( ep_group.rank(), ) + # Reset pending_result before ublocking the async worker + model_state.pending_result = None + result.consumed_event.record() + def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]: """ All-reduce a list of tensors. From 2e2dd8f63254d673fca487c08076c7bbdf995fc3 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 27 Mar 2026 14:19:05 +0000 Subject: [PATCH 19/43] minor comment update Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5cf3b0fe5ba3..5a9ca984bf14 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -119,7 +119,7 @@ async def transfer_run_periodically( # Execute one EPLB layer transfer per model forward pass. Each iteration # of this loop will copy the new set of expert weights into # model_state.expert_buffer, which will be consumed by the main thread in - # EplbState.step() + # move_to_workspace while model_state.rebalanced and layer_idx < current_num_layers: ( is_unchanged, From dff24288fca3999467da0bcddf3dab4741987c45 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 27 Mar 2026 18:27:03 +0000 Subject: [PATCH 20/43] remove window_ready_event Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 6 ------ vllm/distributed/eplb/eplb_state.py | 12 ------------ 2 files changed, 18 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5a9ca984bf14..2f3e8ebac278 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -66,12 +66,6 @@ def run_rebalance_experts( assert model_state.eplb_stats is not None eplb_stats = model_state.eplb_stats - # Wait for the main thread's all-reduce and clone to complete before - # accessing the global_expert_load_window tensor. - assert model_state.window_ready_event is not None - model_state.window_ready_event.wait() - model_state.window_ready_event = None - # Move the global expert load window to CPU for computation. with torch.cuda.stream(cuda_stream): global_expert_load_window = eplb_stats.global_expert_load_window.cpu() diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 3371497199da..d75334ee4167 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -172,11 +172,6 @@ class EplbModelState: """ The buffer to store the expert weights during transfer. """ - window_ready_event: torch.cuda.Event | None - """ - CUDA event recorded after all-reduce and clone on the main thread. - The async worker waits on this before accessing global_expert_load_window. - """ rebalanced: bool """ The flag indicates whether the experts rebalance have been computed. @@ -439,7 +434,6 @@ def add_model( model_name=model_config.model, model=model, expert_buffer=expert_buffer, - window_ready_event=None, rebalanced=False, eplb_stats=None, cuda_device_index=self.cuda_device_index, @@ -750,12 +744,6 @@ def rearrange( num_nodes=num_nodes, num_gpus=num_gpus, ) - # Record event after clone to signal async worker - # that load stats data is ready - sync_event = torch.cuda.Event() - sync_event.record() - eplb_model_state.window_ready_event = sync_event - eplb_model_state.rebalanced = True # Signal async thread to start transferring layers if self.is_async and (not is_profile): From 81bb2dd070a8176ea3ea5dcce7371aa857b321d5 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 27 Mar 2026 19:15:23 +0000 Subject: [PATCH 21/43] cleanup Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- vllm/distributed/eplb/eplb_state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 2f3e8ebac278..1d7f2e62cb98 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -140,7 +140,7 @@ async def transfer_run_periodically( model_state.pending_result = AsyncEPLBLayerResult( layer_idx=layer_idx, - new_physical_to_logical_map=new_physical_to_logical_map, + new_physical_to_logical_map=new_physical_to_logical_map[layer_idx], is_unchanged=is_unchanged, is_received_locally=is_received_locally, recv_metadata=recv_metadata, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index d75334ee4167..13d152fe326e 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -832,7 +832,7 @@ def move_to_workspace( is_unchanged=result.is_unchanged, is_received_locally=result.is_received_locally, recv_metadata=result.recv_metadata, - new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), + new_indices=result.new_physical_to_logical_map.numpy(), ep_rank=ep_group.rank(), ) From 67aadc5c76007766f7c78931d9208dab68b698f5 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 27 Mar 2026 19:16:43 +0000 Subject: [PATCH 22/43] AsyncEplbLayerResult rename Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 4 ++-- vllm/distributed/eplb/eplb_state.py | 6 +++--- vllm/distributed/eplb/rebalance_execute.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 1d7f2e62cb98..68b903a8d464 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -15,7 +15,7 @@ from vllm.logger import init_logger from .eplb_utils import EplbEvent -from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer +from .rebalance_execute import AsyncEplbLayerResult, transfer_layer if TYPE_CHECKING: from .eplb_state import EplbModelState, EplbState @@ -138,7 +138,7 @@ async def transfer_run_periodically( # it. Record is called by the main thread after move_from_buffer(). consumed_event = EplbEvent() - model_state.pending_result = AsyncEPLBLayerResult( + model_state.pending_result = AsyncEplbLayerResult( layer_idx=layer_idx, new_physical_to_logical_map=new_physical_to_logical_map[layer_idx], is_unchanged=is_unchanged, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 13d152fe326e..b83cc0c0ad47 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -48,7 +48,7 @@ from .eplb_utils import EplbEvent from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( - AsyncEPLBLayerResult, + AsyncEplbLayerResult, move_from_buffer, rearrange_expert_weights_inplace, ) @@ -184,7 +184,7 @@ class EplbModelState: """ CUDA device index for the async EPLB worker thread. """ - pending_result: AsyncEPLBLayerResult | None = None + pending_result: AsyncEplbLayerResult | None = None """ Set by the async worker after all writes to expert_buffer are done. Consumed and reset to None by the main thread in move_to_workspace() after the contents of @@ -764,7 +764,7 @@ def start_async_loop( ) def _update_layer_mapping_from_new( - self, model_state: EplbModelState, result: AsyncEPLBLayerResult + self, model_state: EplbModelState, result: AsyncEplbLayerResult ) -> None: layer = result.layer_idx diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 2a211bf1d58d..95a407e5bb4e 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -46,7 +46,7 @@ class RecvMetadata: @dataclass -class AsyncEPLBLayerResult: +class AsyncEplbLayerResult: """ The result of one completed async EPLB layer transfer. """ From b52665916ae8769dde35379f167c2a083f804596 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 30 Mar 2026 16:14:04 +0000 Subject: [PATCH 23/43] comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 4 ++-- vllm/distributed/eplb/eplb_state.py | 4 ++-- vllm/distributed/eplb/eplb_utils.py | 23 +++++++++++++++++++++- vllm/distributed/eplb/rebalance_execute.py | 4 ++-- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 68b903a8d464..b933827df6be 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -14,7 +14,7 @@ from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger -from .eplb_utils import EplbEvent +from .eplb_utils import CpuGpuEvent from .rebalance_execute import AsyncEplbLayerResult, transfer_layer if TYPE_CHECKING: @@ -136,7 +136,7 @@ async def transfer_run_periodically( # This event guarantees that expert_buffer will not be overwritten by # subsequent iterations of this loop until the main thread has consumed # it. Record is called by the main thread after move_from_buffer(). - consumed_event = EplbEvent() + consumed_event = CpuGpuEvent() model_state.pending_result = AsyncEplbLayerResult( layer_idx=layer_idx, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index fcb65f418c77..786a37c533c9 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -45,7 +45,7 @@ from vllm.model_executor.models.interfaces import MixtureOfExperts from .async_worker import start_async_worker -from .eplb_utils import EplbEvent +from .eplb_utils import CpuGpuEvent from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy from .rebalance_execute import ( AsyncEplbLayerResult, @@ -236,7 +236,7 @@ def __init__(self, parallel_config: ParallelConfig, device: torch.device): """ The flag indicates whether the EPLB is running in async mode. """ - self.rearrange_event: EplbEvent = EplbEvent() + self.rearrange_event: CpuGpuEvent = CpuGpuEvent() """ Event to signal when a new rearrangement is needed for the async thread. """ diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 46ff202f6325..107d6bacb31d 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -13,17 +13,38 @@ logger = init_logger(__name__) -class EplbEvent: +class CpuGpuEvent: + """ + Combines a CUDA event with a CPU threading event to enforce record->wait + ordering across two threads. + + CUDA events alone are insufficient for cross-thread synchronization because + waiting on an unrecorded CUDA event is a no-op. The wait will return + immediately instead of blocking. This class adds a threading.Event so + that the waiting thread blocks on the CPU side until record() has + actually been called, at which point the CUDA event is guaranteed to be + in-flight and event.wait() will correctly synchronize the GPU stream. + """ + def __init__(self): self._event = torch.cuda.Event() self._recorded = threading.Event() def wait(self, stream: torch.cuda.Stream | None = None): + """ + Blocks the calling thread until record finishes. Used to guarantee that the + record kernel is called before wait. + + Should only be called by the Async Eplb thread + """ self._recorded.wait() self._event.wait(stream) self._recorded.clear() def record(self, stream: torch.cuda.Stream | None = None): + """ + Unblocks the waiting thread after calling event.record(). + """ assert not self._recorded.is_set() self._event.record(stream) self._recorded.set() diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 95a407e5bb4e..bea360602933 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -19,7 +19,7 @@ get_global_rank, ) -from vllm.distributed.eplb.eplb_utils import EplbEvent +from vllm.distributed.eplb.eplb_utils import CpuGpuEvent from vllm.distributed.parallel_state import get_ep_group from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger @@ -64,7 +64,7 @@ class AsyncEplbLayerResult: """Per-physical-expert flag: weight was received on this rank.""" recv_metadata: RecvMetadata """Metadata describing what was received during transfer_layer.""" - consumed_event: EplbEvent + consumed_event: CpuGpuEvent """ Created by the async worker before publishing this result. The main thread calls consumed_event.record() after move_from_buffer() completes, which From 1cd89484e2f08ded762e6288303a1ee68eeb0d4e Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 30 Mar 2026 16:14:59 +0000 Subject: [PATCH 24/43] comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 107d6bacb31d..11f218db2c66 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -35,7 +35,7 @@ def wait(self, stream: torch.cuda.Stream | None = None): Blocks the calling thread until record finishes. Used to guarantee that the record kernel is called before wait. - Should only be called by the Async Eplb thread + Should only be called by the Async Eplb thread. """ self._recorded.wait() self._event.wait(stream) @@ -44,6 +44,8 @@ def wait(self, stream: torch.cuda.Stream | None = None): def record(self, stream: torch.cuda.Stream | None = None): """ Unblocks the waiting thread after calling event.record(). + + Should only be called by the main thread. """ assert not self._recorded.is_set() self._event.record(stream) From cea868a27c121e25b4eeb2ed521cda983e4129a7 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 30 Mar 2026 16:38:17 +0000 Subject: [PATCH 25/43] bugfix Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index b933827df6be..b9f40a5cc1e4 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -90,7 +90,7 @@ async def transfer_run_periodically( is_profile: bool = False, ) -> None: while True: - state.rearrange_event.wait() + state.rearrange_event.wait(stream=cuda_stream) logger.info("async worker woke up for EPLB transfer") assert state.is_async From 26751decf45aca0e04e818478bdeded27df600d7 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 15:45:15 +0000 Subject: [PATCH 26/43] bugfix Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 11f218db2c66..398cbc2669cf 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -48,6 +48,7 @@ def record(self, stream: torch.cuda.Stream | None = None): Should only be called by the main thread. """ assert not self._recorded.is_set() + self._event = torch.cuda.Event() self._event.record(stream) self._recorded.set() From d00793eb0d700b58f100468cfdb43d718db07e55 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 20:17:51 +0000 Subject: [PATCH 27/43] add event unit test Signed-off-by: Sage Moore --- tests/distributed/test_eplb_events.py | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/distributed/test_eplb_events.py diff --git a/tests/distributed/test_eplb_events.py b/tests/distributed/test_eplb_events.py new file mode 100644 index 000000000000..40323dc89809 --- /dev/null +++ b/tests/distributed/test_eplb_events.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +import time + +import torch + +from vllm.distributed.eplb.eplb_utils import CpuGpuEvent + + +def test_wait_blocks_until_record(): + event = CpuGpuEvent() + record_stream = torch.cuda.Stream() + wait_stream = torch.cuda.Stream() + wait_returned = threading.Event() + + def waiter(): + event.wait(stream=wait_stream) + wait_returned.set() + + t = threading.Thread(target=waiter) + t.start() + + time.sleep(0.05) + assert not wait_returned.is_set(), "wait() returned before record() was called" + + event.record(stream=record_stream) + t.join(timeout=5.0) + + assert not event._recorded.is_set() + + +def test_reuse_across_multiple_cycles(): + wrapper = CpuGpuEvent() + record_stream = torch.cuda.Stream() + wait_stream = torch.cuda.Stream() + NUM_CYCLES = 8 + completed_cycles = [] + barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)] + + def waiter(): + for i in range(NUM_CYCLES): + wrapper.wait(stream=wait_stream) + completed_cycles.append(True) + barriers[i].wait() + + t = threading.Thread(target=waiter) + t.start() + + for i in range(NUM_CYCLES): + wrapper.record(stream=record_stream) + barriers[i].wait() + + t.join(timeout=10.0) + assert len(completed_cycles) == NUM_CYCLES + + +def test_producer_consumer(): + """ + This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared GPU + tensor on multiple CPU threads. + """ + worker_stream = torch.cuda.Stream() + # Create a single element counter that will be shared between two threads + buf = torch.zeros(1, device="cuda") + NUM_ROUNDS = 5 + + ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)] + events = [CpuGpuEvent() for _ in range(NUM_ROUNDS)] + errors: list[str] = [] + + # For each round, the worker thread (writer) sets the counter in buf and waits for + # the main thread to read it. + def worker(): + for i in range(NUM_ROUNDS): + if i > 0: + events[i - 1].wait(stream=worker_stream) + + with torch.cuda.stream(worker_stream): + buf.fill_(float(i + 1)) + + worker_stream.synchronize() + ready_cpu[i].set() + + t = threading.Thread(target=worker) + t.start() + + for i in range(NUM_ROUNDS): + ready_cpu[i].wait() + snapshot = buf.clone() + events[i].record() + val = snapshot.item() + if val != float(i + 1): + errors.append(f"round {i}: expected {i + 1:.1f}, got {val:.1f}") + + t.join(timeout=10.0) + assert not errors, f"Buffer ordering errors: {errors}" From 0610ad4db2894d558e4d73f4d56ca4e8a4287792 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 22:09:39 +0000 Subject: [PATCH 28/43] add async worker unit tests Signed-off-by: Sage Moore --- tests/distributed/test_eplb_async.py | 131 +++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/distributed/test_eplb_async.py diff --git a/tests/distributed/test_eplb_async.py b/tests/distributed/test_eplb_async.py new file mode 100644 index 000000000000..471f6c4eb0ad --- /dev/null +++ b/tests/distributed/test_eplb_async.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio +import threading +import time +from types import SimpleNamespace +from unittest import mock + +import torch + +from vllm.distributed.eplb.async_worker import transfer_run_periodically +from vllm.distributed.eplb.eplb_state import EplbModelState, EplbStats +from vllm.distributed.eplb.eplb_utils import CpuGpuEvent + + +def make_model_state(physical_to_logical_map: torch.Tensor) -> EplbModelState: + """Build a minimal EplbModelState with the given physical_to_logical_map.""" + num_layers, num_physical = physical_to_logical_map.shape + model = SimpleNamespace( + num_moe_layers=num_layers, + expert_weights=[[torch.empty(1, device="cuda")] for _ in range(num_layers)], + model_name="test", + ) + return EplbModelState( + physical_to_logical_map=physical_to_logical_map.clone(), + logical_to_physical_map=torch.zeros( + num_layers, num_physical, 1, dtype=torch.long, device="cuda" + ), + logical_replica_count=torch.ones( + num_layers, num_physical, dtype=torch.long, device="cuda" + ), + expert_load_pass=torch.zeros(num_layers, num_physical, device="cuda"), + expert_load_window=torch.zeros(1, num_layers, num_physical, device="cuda"), + model_name="test", + model=model, + expert_buffer=[torch.empty(1, device="cuda") for _ in range(num_layers)], + rebalanced=False, + eplb_stats=None, + cuda_device_index=torch.accelerator.current_device_index(), + ) + + +def make_eplb_stats(num_layers: int, num_physical: int) -> EplbStats: + return EplbStats( + global_expert_load_window=torch.zeros( + 1, num_layers, num_physical, device="cuda" + ), + num_replicas=1, + num_groups=1, + num_nodes=1, + num_gpus=1, + ) + + +def make_eplb_state(model_state: EplbModelState) -> SimpleNamespace: + """Build a minimal EplbState-like namespace with an identity rebalance policy.""" + return SimpleNamespace( + rearrange_event=CpuGpuEvent(), + is_async=True, + model_states={"model": model_state}, + # Identity policy: return the current map unchanged. + policy=SimpleNamespace( + rebalance_experts=lambda load, nr, ng, nn, ngpu, p2l: p2l.clone() + ), + ) + + +def start_worker_thread(eplb_state: SimpleNamespace) -> threading.Thread: + """Run transfer_run_periodically in a daemon thread.""" + + def run() -> None: + stream = torch.cuda.Stream() + loop = asyncio.new_event_loop() + loop.run_until_complete( + transfer_run_periodically( + state=eplb_state, + eplb_group=None, # not used — transfer_layer is mocked + cuda_stream=stream, + ) + ) + + t = threading.Thread(target=run, daemon=True) + t.start() + return t + + +def test_worker_snapshots_map_and_load_window_at_wake_time(): + NUM_LAYERS = 1 + NUM_PHYSICAL_EXPERTS = 4 + + p2l_v1 = torch.arange(NUM_PHYSICAL_EXPERTS, device="cuda").unsqueeze(0) + p2l_v2 = torch.tensor([[3, 2, 1, 0]], device="cuda") + + # A recognisable non-zero load window to distinguish from the default zeros. + load_window = torch.full((1, NUM_LAYERS, NUM_PHYSICAL_EXPERTS), 42.0, device="cuda") + + model_state = make_model_state(p2l_v1) + eplb_state = make_eplb_state(model_state) + + captured_old_indices = [] + captured_load_windows = [] + transfer_called = threading.Event() + + def capturing_rebalance(load, nr, ng, nn, ngpu, p2l): + captured_load_windows.append(load.clone()) + return p2l.clone() + + eplb_state.policy.rebalance_experts = capturing_rebalance + + async def mock_transfer(**kwargs): + captured_old_indices.append(kwargs["old_layer_indices"].clone()) + transfer_called.set() + return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() + + with mock.patch("vllm.distributed.eplb.async_worker.transfer_layer", mock_transfer): + start_worker_thread(eplb_state) + time.sleep(5) + + # Update the map to V2 and set a recognisable load window before + # unblocking the async worker. + model_state.physical_to_logical_map.copy_(p2l_v2) + model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) + model_state.eplb_stats.global_expert_load_window = load_window + model_state.rebalanced = True + eplb_state.rearrange_event.record() + + assert transfer_called.wait(timeout=5.0), "transfer_layer was not called" + + assert torch.equal(captured_old_indices[0], p2l_v2[0].cpu()) + assert torch.equal(captured_load_windows[0], load_window.cpu()) From 5d0f02e5c666efceae35a0d024ca7012d118163c Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 22:27:17 +0000 Subject: [PATCH 29/43] move move_to_workspace out of EplbState Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_state.py | 66 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 786a37c533c9..ebfbd1bd7c66 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -549,7 +549,7 @@ def step( eplb_model_state.pending_result is not None and self._all_ranks_result_ready(eplb_model_state) ): - self.move_to_workspace( + _move_to_workspace( model_state=eplb_model_state, ep_group=ep_group, ) @@ -749,38 +749,6 @@ def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: all_reduce(flag, group=device_group) return int(flag.item()) == device_group.size() - def move_to_workspace( - self, - model_state: EplbModelState, - ep_group: ProcessGroup, - ) -> None: - result = model_state.pending_result - assert result is not None - move_from_buffer( - expert_weights=model_state.model.expert_weights[result.layer_idx], - expert_weights_buffers=model_state.expert_buffer, - is_unchanged=result.is_unchanged, - is_received_locally=result.is_received_locally, - recv_metadata=result.recv_metadata, - new_indices=result.new_physical_to_logical_map.numpy(), - ep_rank=ep_group.rank(), - ) - - _commit_eplb_maps_for_layer( - model_state, - new_physical_to_logical_map=result.new_physical_to_logical_map, - layer=result.layer_idx, - ) - logger.debug( - "model %s successfully move_to_workspace layer %d", - model_state.model_name, - ep_group.rank(), - ) - - # Reset pending_result before ublocking the async worker - model_state.pending_result = None - result.consumed_event.record() - def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]: """ All-reduce a list of tensors. @@ -1066,3 +1034,35 @@ def _commit_eplb_maps( src = new_replica_count dst = model_state.logical_replica_count dst.copy_(src, non_blocking=True) + + +def _move_to_workspace( + model_state: EplbModelState, + ep_group: ProcessGroup, +) -> None: + result = model_state.pending_result + assert result is not None + move_from_buffer( + expert_weights=model_state.model.expert_weights[result.layer_idx], + expert_weights_buffers=model_state.expert_buffer, + is_unchanged=result.is_unchanged, + is_received_locally=result.is_received_locally, + recv_metadata=result.recv_metadata, + new_indices=result.new_physical_to_logical_map.numpy(), + ep_rank=ep_group.rank(), + ) + + _commit_eplb_maps_for_layer( + model_state, + new_physical_to_logical_map=result.new_physical_to_logical_map, + layer=result.layer_idx, + ) + logger.debug( + "model %s successfully move_to_workspace layer %d", + model_state.model_name, + ep_group.rank(), + ) + + # Reset pending_result before unblocking the async worker + model_state.pending_result = None + result.consumed_event.record() From bf7b1547d34e7101d79d20e5a0aaf3e48c7f18b1 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 22:51:03 +0000 Subject: [PATCH 30/43] more unit tests Signed-off-by: Sage Moore --- tests/distributed/test_eplb_async.py | 47 +++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/distributed/test_eplb_async.py b/tests/distributed/test_eplb_async.py index 471f6c4eb0ad..940f14b1a49c 100644 --- a/tests/distributed/test_eplb_async.py +++ b/tests/distributed/test_eplb_async.py @@ -10,7 +10,11 @@ import torch from vllm.distributed.eplb.async_worker import transfer_run_periodically -from vllm.distributed.eplb.eplb_state import EplbModelState, EplbStats +from vllm.distributed.eplb.eplb_state import ( + EplbModelState, + EplbStats, + _move_to_workspace, +) from vllm.distributed.eplb.eplb_utils import CpuGpuEvent @@ -129,3 +133,44 @@ async def mock_transfer(**kwargs): assert torch.equal(captured_old_indices[0], p2l_v2[0].cpu()) assert torch.equal(captured_load_windows[0], load_window.cpu()) + + +def test_consumed_event_handshake(): + """ + The worker blocks on consumed_event.wait() after publishing pending_result. + _move_to_workspace must record consumed_event to unblock it. + """ + NUM_LAYERS = 1 + NUM_PHYSICAL_EXPERTS = 4 + + p2l = torch.arange(NUM_PHYSICAL_EXPERTS, device="cuda").unsqueeze(0) + model_state = make_model_state(p2l) + eplb_state = make_eplb_state(model_state) + ep_group = SimpleNamespace(rank=lambda: 0) + + async def mock_transfer(**kwargs): + return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() + + with ( + mock.patch("vllm.distributed.eplb.async_worker.transfer_layer", mock_transfer), + mock.patch("vllm.distributed.eplb.eplb_state.move_from_buffer"), + ): + start_worker_thread(eplb_state) + time.sleep(5) + + model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) + model_state.rebalanced = True + eplb_state.rearrange_event.record() + + # Poll until the worker publishes pending_result. + deadline = time.monotonic() + 5.0 + while model_state.pending_result is None: + assert time.monotonic() < deadline, "worker never published pending_result" + time.sleep(0.001) + + # Worker is now blocked on consumed_event.wait() — event not yet recorded. + assert not model_state.pending_result.consumed_event._recorded.is_set() + + _move_to_workspace(model_state, ep_group) + + assert model_state.pending_result is None From a12732d6f5dcbaffef7ef9636e3f80295c7c2855 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 31 Mar 2026 23:34:46 +0000 Subject: [PATCH 31/43] bugfix Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index b9f40a5cc1e4..c737f71fc002 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -140,7 +140,7 @@ async def transfer_run_periodically( model_state.pending_result = AsyncEplbLayerResult( layer_idx=layer_idx, - new_physical_to_logical_map=new_physical_to_logical_map[layer_idx], + new_physical_to_logical_map=new_physical_to_logical_map, is_unchanged=is_unchanged, is_received_locally=is_received_locally, recv_metadata=recv_metadata, From 2d69c8f2472ea9587c16d1f5f9116c3e602c7645 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 2 Apr 2026 13:22:16 +0000 Subject: [PATCH 32/43] misc bugfixes Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_state.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index ebfbd1bd7c66..6886603fa2c7 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -544,11 +544,10 @@ def step( self.expert_rearrangement_step += 1 if self.is_async: + # Run _move_to_workspace if all ranks have finished transferring the + # new weights to the intermediate buffer for eplb_model_state in self.model_states.values(): - if ( - eplb_model_state.pending_result is not None - and self._all_ranks_result_ready(eplb_model_state) - ): + if self._all_ranks_result_ready(eplb_model_state): _move_to_workspace( model_state=eplb_model_state, ep_group=ep_group, @@ -1048,7 +1047,7 @@ def _move_to_workspace( is_unchanged=result.is_unchanged, is_received_locally=result.is_received_locally, recv_metadata=result.recv_metadata, - new_indices=result.new_physical_to_logical_map.numpy(), + new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), ep_rank=ep_group.rank(), ) @@ -1063,6 +1062,9 @@ def _move_to_workspace( ep_group.rank(), ) + if result.layer_idx == model_state.model.num_moe_layers - 1: + model_state.rebalanced = False + # Reset pending_result before unblocking the async worker model_state.pending_result = None result.consumed_event.record() From e812dd82e0f1421fd51972830e5d458b10ee0060 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 2 Apr 2026 13:29:39 +0000 Subject: [PATCH 33/43] comment update Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 398cbc2669cf..3dda48b3d622 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -21,9 +21,9 @@ class CpuGpuEvent: CUDA events alone are insufficient for cross-thread synchronization because waiting on an unrecorded CUDA event is a no-op. The wait will return immediately instead of blocking. This class adds a threading.Event so - that the waiting thread blocks on the CPU side until record() has - actually been called, at which point the CUDA event is guaranteed to be - in-flight and event.wait() will correctly synchronize the GPU stream. + that the waiting thread blocks on the CPU side until record() is called, at + which point the CUDA event is guaranteed to be in-flight and event.wait() will + correctly synchronize the GPU stream. """ def __init__(self): From d61149cd7989c56bf351e7f2f466b8407c3748b1 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 2 Apr 2026 13:35:38 +0000 Subject: [PATCH 34/43] comment update Signed-off-by: Sage Moore --- vllm/distributed/eplb/rebalance_execute.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index bea360602933..cced4ebf7aab 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -66,9 +66,10 @@ class AsyncEplbLayerResult: """Metadata describing what was received during transfer_layer.""" consumed_event: CpuGpuEvent """ - Created by the async worker before publishing this result. The main thread - calls consumed_event.record() after move_from_buffer() completes, which - unblocks the async worker's consumed_event.wait() call. + Event used to synchronize access to the intermediate buffer. The async worker calls + wait() after it finishes transferring weights to the intermediate buffer. The main + thread calls record() after it finishes transferring weights out of the intermediate + buffer in _move_to_workspace() """ From 0b64b6ac6276bc4a80d49d709703c5093a565cb5 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 2 Apr 2026 17:54:44 +0000 Subject: [PATCH 35/43] misc refactoring Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- vllm/distributed/eplb/eplb_state.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 11dec2981f0e..b32e22975a0b 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -143,7 +143,7 @@ async def transfer_run_periodically( model_state.pending_result = AsyncEplbLayerResult( layer_idx=layer_idx, - new_physical_to_logical_map=new_physical_to_logical_map, + new_physical_to_logical_map=new_physical_to_logical_map[layer_idx], is_unchanged=is_unchanged, is_received_locally=is_received_locally, recv_metadata=recv_metadata, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index a7b36db447e3..aec4917766ab 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -1064,7 +1064,7 @@ def _commit_eplb_maps_for_layer( """ # Commit physical_to_logical_map - src = new_physical_to_logical_map[layer] + src = new_physical_to_logical_map dst = model_state.physical_to_logical_map[layer] assert src.shape == dst.shape, ( "The number of physical experts must stay the same while running Async EPLB. " @@ -1137,7 +1137,7 @@ def _move_to_workspace( is_unchanged=result.is_unchanged, is_received_locally=result.is_received_locally, recv_metadata=result.recv_metadata, - new_indices=result.new_physical_to_logical_map[result.layer_idx].numpy(), + new_indices=result.new_physical_to_logical_map.numpy(), ep_rank=ep_group.rank(), ) From 3545c13dbb47667b92df1714e271b1375844caee Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 3 Apr 2026 00:08:34 +0000 Subject: [PATCH 36/43] misc refactoring Signed-off-by: Sage Moore --- tests/distributed/test_eplb_async.py | 35 ++++++++++++++++++---------- tests/distributed/test_eplb_utils.py | 4 ++-- vllm/distributed/eplb/eplb_state.py | 8 +++---- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/tests/distributed/test_eplb_async.py b/tests/distributed/test_eplb_async.py index 940f14b1a49c..4b779561db9a 100644 --- a/tests/distributed/test_eplb_async.py +++ b/tests/distributed/test_eplb_async.py @@ -42,6 +42,7 @@ def make_model_state(physical_to_logical_map: torch.Tensor) -> EplbModelState: rebalanced=False, eplb_stats=None, cuda_device_index=torch.accelerator.current_device_index(), + communicator=mock.MagicMock(), ) @@ -71,7 +72,7 @@ def make_eplb_state(model_state: EplbModelState) -> SimpleNamespace: def start_worker_thread(eplb_state: SimpleNamespace) -> threading.Thread: - """Run transfer_run_periodically in a daemon thread.""" + """Run transfer_run_periodically in a background thread.""" def run() -> None: stream = torch.cuda.Stream() @@ -93,13 +94,14 @@ def test_worker_snapshots_map_and_load_window_at_wake_time(): NUM_LAYERS = 1 NUM_PHYSICAL_EXPERTS = 4 - p2l_v1 = torch.arange(NUM_PHYSICAL_EXPERTS, device="cuda").unsqueeze(0) - p2l_v2 = torch.tensor([[3, 2, 1, 0]], device="cuda") + physical_to_logical_map_v1 = torch.arange( + NUM_PHYSICAL_EXPERTS, device="cuda" + ).unsqueeze(0) + physical_to_logical_map_v2 = torch.tensor([[3, 2, 1, 0]], device="cuda") - # A recognisable non-zero load window to distinguish from the default zeros. load_window = torch.full((1, NUM_LAYERS, NUM_PHYSICAL_EXPERTS), 42.0, device="cuda") - model_state = make_model_state(p2l_v1) + model_state = make_model_state(physical_to_logical_map_v1) eplb_state = make_eplb_state(model_state) captured_old_indices = [] @@ -112,7 +114,9 @@ def capturing_rebalance(load, nr, ng, nn, ngpu, p2l): eplb_state.policy.rebalance_experts = capturing_rebalance + # Replacement for transfer_layer that clones the physical_to_logical_map async def mock_transfer(**kwargs): + # old_layer_indices is the physical_to_logical_map captured_old_indices.append(kwargs["old_layer_indices"].clone()) transfer_called.set() return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() @@ -123,7 +127,7 @@ async def mock_transfer(**kwargs): # Update the map to V2 and set a recognisable load window before # unblocking the async worker. - model_state.physical_to_logical_map.copy_(p2l_v2) + model_state.physical_to_logical_map.copy_(physical_to_logical_map_v2) model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) model_state.eplb_stats.global_expert_load_window = load_window model_state.rebalanced = True @@ -131,7 +135,8 @@ async def mock_transfer(**kwargs): assert transfer_called.wait(timeout=5.0), "transfer_layer was not called" - assert torch.equal(captured_old_indices[0], p2l_v2[0].cpu()) + # Assert that the physical_to_logical_map and global_expert_load_window are + assert torch.equal(captured_old_indices[0], physical_to_logical_map_v2[0].cpu()) assert torch.equal(captured_load_windows[0], load_window.cpu()) @@ -143,14 +148,17 @@ def test_consumed_event_handshake(): NUM_LAYERS = 1 NUM_PHYSICAL_EXPERTS = 4 - p2l = torch.arange(NUM_PHYSICAL_EXPERTS, device="cuda").unsqueeze(0) - model_state = make_model_state(p2l) + physical_to_logical_map = torch.arange( + NUM_PHYSICAL_EXPERTS, device="cuda" + ).unsqueeze(0) + model_state = make_model_state(physical_to_logical_map) eplb_state = make_eplb_state(model_state) - ep_group = SimpleNamespace(rank=lambda: 0) + # Create an empty stub for transfer_layer async def mock_transfer(**kwargs): return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() + # Avoid any GPU -> GPU transfers by mocking out transfer_layer and move_from_buffer with ( mock.patch("vllm.distributed.eplb.async_worker.transfer_layer", mock_transfer), mock.patch("vllm.distributed.eplb.eplb_state.move_from_buffer"), @@ -160,6 +168,8 @@ async def mock_transfer(**kwargs): model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) model_state.rebalanced = True + + # Unblock the async thread eplb_state.rearrange_event.record() # Poll until the worker publishes pending_result. @@ -168,9 +178,10 @@ async def mock_transfer(**kwargs): assert time.monotonic() < deadline, "worker never published pending_result" time.sleep(0.001) - # Worker is now blocked on consumed_event.wait() — event not yet recorded. + # Assert that the worker is now blocked on consumed_event.wait(). assert not model_state.pending_result.consumed_event._recorded.is_set() - _move_to_workspace(model_state, ep_group) + _move_to_workspace(model_state, ep_rank=0) assert model_state.pending_result is None + assert not model_state.rebalanced diff --git a/tests/distributed/test_eplb_utils.py b/tests/distributed/test_eplb_utils.py index 8b287244b742..53a4ce21af2e 100644 --- a/tests/distributed/test_eplb_utils.py +++ b/tests/distributed/test_eplb_utils.py @@ -80,7 +80,7 @@ def test_commit_eplb_maps_for_layer_logical_padding(): .contiguous() ) layer = 0 - _commit_eplb_maps_for_layer(model_state, new_phy2log, layer) + _commit_eplb_maps_for_layer(model_state, new_phy2log[layer], layer) assert torch.all(model_state.logical_to_physical_map[layer, :, 2] == -1) @@ -143,7 +143,7 @@ def test_commit_eplb_maps_for_layer(): ) new_logcnt = torch.tensor([[2, 1, 1], [1, 2, 1]], dtype=torch.long) - _commit_eplb_maps_for_layer(model_state, new_phy2log, layer=0) + _commit_eplb_maps_for_layer(model_state, new_phy2log[0], layer=0) # Layer 0 updated assert torch.equal(model_state.physical_to_logical_map[0], new_phy2log[0]) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index aec4917766ab..4e2ad4548e80 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -573,7 +573,7 @@ def step( if self._all_ranks_result_ready(eplb_model_state): _move_to_workspace( model_state=eplb_model_state, - ep_group=ep_group, + ep_rank=ep_group.rank(), ) if self.expert_rearrangement_step >= self.expert_rearrangement_step_interval: @@ -1127,7 +1127,7 @@ def _commit_eplb_maps( def _move_to_workspace( model_state: EplbModelState, - ep_group: ProcessGroup, + ep_rank: int, ) -> None: result = model_state.pending_result assert result is not None @@ -1138,7 +1138,7 @@ def _move_to_workspace( is_received_locally=result.is_received_locally, recv_metadata=result.recv_metadata, new_indices=result.new_physical_to_logical_map.numpy(), - ep_rank=ep_group.rank(), + ep_rank=ep_rank, ) _commit_eplb_maps_for_layer( @@ -1149,7 +1149,7 @@ def _move_to_workspace( logger.debug( "model %s successfully move_to_workspace layer %d", model_state.model_name, - ep_group.rank(), + ep_rank, ) if result.layer_idx == model_state.model.num_moe_layers - 1: From f2b7647da8d1bbd48d2c76f30a373afba271608c Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 3 Apr 2026 00:09:42 +0000 Subject: [PATCH 37/43] remove extra tests Signed-off-by: Sage Moore --- tests/distributed/test_eplb_async.py | 187 --------------------------- 1 file changed, 187 deletions(-) delete mode 100644 tests/distributed/test_eplb_async.py diff --git a/tests/distributed/test_eplb_async.py b/tests/distributed/test_eplb_async.py deleted file mode 100644 index 4b779561db9a..000000000000 --- a/tests/distributed/test_eplb_async.py +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import asyncio -import threading -import time -from types import SimpleNamespace -from unittest import mock - -import torch - -from vllm.distributed.eplb.async_worker import transfer_run_periodically -from vllm.distributed.eplb.eplb_state import ( - EplbModelState, - EplbStats, - _move_to_workspace, -) -from vllm.distributed.eplb.eplb_utils import CpuGpuEvent - - -def make_model_state(physical_to_logical_map: torch.Tensor) -> EplbModelState: - """Build a minimal EplbModelState with the given physical_to_logical_map.""" - num_layers, num_physical = physical_to_logical_map.shape - model = SimpleNamespace( - num_moe_layers=num_layers, - expert_weights=[[torch.empty(1, device="cuda")] for _ in range(num_layers)], - model_name="test", - ) - return EplbModelState( - physical_to_logical_map=physical_to_logical_map.clone(), - logical_to_physical_map=torch.zeros( - num_layers, num_physical, 1, dtype=torch.long, device="cuda" - ), - logical_replica_count=torch.ones( - num_layers, num_physical, dtype=torch.long, device="cuda" - ), - expert_load_pass=torch.zeros(num_layers, num_physical, device="cuda"), - expert_load_window=torch.zeros(1, num_layers, num_physical, device="cuda"), - model_name="test", - model=model, - expert_buffer=[torch.empty(1, device="cuda") for _ in range(num_layers)], - rebalanced=False, - eplb_stats=None, - cuda_device_index=torch.accelerator.current_device_index(), - communicator=mock.MagicMock(), - ) - - -def make_eplb_stats(num_layers: int, num_physical: int) -> EplbStats: - return EplbStats( - global_expert_load_window=torch.zeros( - 1, num_layers, num_physical, device="cuda" - ), - num_replicas=1, - num_groups=1, - num_nodes=1, - num_gpus=1, - ) - - -def make_eplb_state(model_state: EplbModelState) -> SimpleNamespace: - """Build a minimal EplbState-like namespace with an identity rebalance policy.""" - return SimpleNamespace( - rearrange_event=CpuGpuEvent(), - is_async=True, - model_states={"model": model_state}, - # Identity policy: return the current map unchanged. - policy=SimpleNamespace( - rebalance_experts=lambda load, nr, ng, nn, ngpu, p2l: p2l.clone() - ), - ) - - -def start_worker_thread(eplb_state: SimpleNamespace) -> threading.Thread: - """Run transfer_run_periodically in a background thread.""" - - def run() -> None: - stream = torch.cuda.Stream() - loop = asyncio.new_event_loop() - loop.run_until_complete( - transfer_run_periodically( - state=eplb_state, - eplb_group=None, # not used — transfer_layer is mocked - cuda_stream=stream, - ) - ) - - t = threading.Thread(target=run, daemon=True) - t.start() - return t - - -def test_worker_snapshots_map_and_load_window_at_wake_time(): - NUM_LAYERS = 1 - NUM_PHYSICAL_EXPERTS = 4 - - physical_to_logical_map_v1 = torch.arange( - NUM_PHYSICAL_EXPERTS, device="cuda" - ).unsqueeze(0) - physical_to_logical_map_v2 = torch.tensor([[3, 2, 1, 0]], device="cuda") - - load_window = torch.full((1, NUM_LAYERS, NUM_PHYSICAL_EXPERTS), 42.0, device="cuda") - - model_state = make_model_state(physical_to_logical_map_v1) - eplb_state = make_eplb_state(model_state) - - captured_old_indices = [] - captured_load_windows = [] - transfer_called = threading.Event() - - def capturing_rebalance(load, nr, ng, nn, ngpu, p2l): - captured_load_windows.append(load.clone()) - return p2l.clone() - - eplb_state.policy.rebalance_experts = capturing_rebalance - - # Replacement for transfer_layer that clones the physical_to_logical_map - async def mock_transfer(**kwargs): - # old_layer_indices is the physical_to_logical_map - captured_old_indices.append(kwargs["old_layer_indices"].clone()) - transfer_called.set() - return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() - - with mock.patch("vllm.distributed.eplb.async_worker.transfer_layer", mock_transfer): - start_worker_thread(eplb_state) - time.sleep(5) - - # Update the map to V2 and set a recognisable load window before - # unblocking the async worker. - model_state.physical_to_logical_map.copy_(physical_to_logical_map_v2) - model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) - model_state.eplb_stats.global_expert_load_window = load_window - model_state.rebalanced = True - eplb_state.rearrange_event.record() - - assert transfer_called.wait(timeout=5.0), "transfer_layer was not called" - - # Assert that the physical_to_logical_map and global_expert_load_window are - assert torch.equal(captured_old_indices[0], physical_to_logical_map_v2[0].cpu()) - assert torch.equal(captured_load_windows[0], load_window.cpu()) - - -def test_consumed_event_handshake(): - """ - The worker blocks on consumed_event.wait() after publishing pending_result. - _move_to_workspace must record consumed_event to unblock it. - """ - NUM_LAYERS = 1 - NUM_PHYSICAL_EXPERTS = 4 - - physical_to_logical_map = torch.arange( - NUM_PHYSICAL_EXPERTS, device="cuda" - ).unsqueeze(0) - model_state = make_model_state(physical_to_logical_map) - eplb_state = make_eplb_state(model_state) - - # Create an empty stub for transfer_layer - async def mock_transfer(**kwargs): - return mock.MagicMock(), mock.MagicMock(), mock.MagicMock() - - # Avoid any GPU -> GPU transfers by mocking out transfer_layer and move_from_buffer - with ( - mock.patch("vllm.distributed.eplb.async_worker.transfer_layer", mock_transfer), - mock.patch("vllm.distributed.eplb.eplb_state.move_from_buffer"), - ): - start_worker_thread(eplb_state) - time.sleep(5) - - model_state.eplb_stats = make_eplb_stats(NUM_LAYERS, NUM_PHYSICAL_EXPERTS) - model_state.rebalanced = True - - # Unblock the async thread - eplb_state.rearrange_event.record() - - # Poll until the worker publishes pending_result. - deadline = time.monotonic() + 5.0 - while model_state.pending_result is None: - assert time.monotonic() < deadline, "worker never published pending_result" - time.sleep(0.001) - - # Assert that the worker is now blocked on consumed_event.wait(). - assert not model_state.pending_result.consumed_event._recorded.is_set() - - _move_to_workspace(model_state, ep_rank=0) - - assert model_state.pending_result is None - assert not model_state.rebalanced From 894a1effb8efaaf03c524f24f783b45f1c912044 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 3 Apr 2026 00:32:39 +0000 Subject: [PATCH 38/43] fix extra ars Signed-off-by: Sage Moore --- vllm/distributed/eplb/eplb_state.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 4e2ad4548e80..10285ddd1489 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -570,7 +570,11 @@ def step( # Run _move_to_workspace if all ranks have finished transferring the # new weights to the intermediate buffer for eplb_model_state in self.model_states.values(): - if self._all_ranks_result_ready(eplb_model_state): + # rebalanced must remain consistent amongst all ranks otherwise this + # all_reduce will hang + if eplb_model_state.rebalanced and self._all_ranks_result_ready( + eplb_model_state + ): _move_to_workspace( model_state=eplb_model_state, ep_rank=ep_group.rank(), From 2f1c851b77661507ec677dc3772e187adcbd8d23 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Fri, 3 Apr 2026 00:39:31 +0000 Subject: [PATCH 39/43] cleanup Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- vllm/distributed/eplb/eplb_state.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index b32e22975a0b..9ffa2dbcaca6 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -133,7 +133,7 @@ async def transfer_run_periodically( ) # Wait until all writes to expert_buffer have finished before making the - # AsynEPLBLayerResult visible to the main thread. + # AsynEplbLayerResult visible to the main thread. cuda_stream.synchronize() # This event guarantees that expert_buffer will not be overwritten by diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 10285ddd1489..47e2b1902c23 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -1150,11 +1150,6 @@ def _move_to_workspace( new_physical_to_logical_map=result.new_physical_to_logical_map, layer=result.layer_idx, ) - logger.debug( - "model %s successfully move_to_workspace layer %d", - model_state.model_name, - ep_rank, - ) if result.layer_idx == model_state.model.num_moe_layers - 1: model_state.rebalanced = False From 99bf9c344579df7f8b3726c8eecda9542892fce3 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Mon, 6 Apr 2026 23:56:03 +0000 Subject: [PATCH 40/43] comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 7 ++++--- vllm/distributed/eplb/rebalance_execute.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 9ffa2dbcaca6..d2e4fe1021ad 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -100,7 +100,8 @@ async def transfer_run_periodically( model_state.communicator.set_stream(cuda_stream) current_num_layers = model_state.model.num_moe_layers - # Snapshot the physical_to_logical_map and copy it to CPU + # Snapshot the physical_to_logical_map (synchronized with + # rearrange_event) and copy it to CPU with torch.cuda.stream(cuda_stream): physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() @@ -150,8 +151,8 @@ async def transfer_run_periodically( consumed_event=consumed_event, ) - # Block the this thread until the main thread finishes copying - # model_state.expert_buffer into + # Block this thread until the main thread and main stream + # finish copying model_state.expert_buffer into # model_state.model.expert_weights[layer_idx] consumed_event.wait(stream=cuda_stream) assert model_state.pending_result is None diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index 648c25184f1d..d85ad075eced 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -49,7 +49,7 @@ class AsyncEplbLayerResult: new_physical_to_logical_map: torch.Tensor """ New physical→logical mapping for all layers, on CPU. - Shape: (num_moe_layers, num_physical_experts) + Shape: (num_physical_experts) """ is_unchanged: np.ndarray """Per-physical-expert flag: weight was not moved during transfer.""" From f2d3bcf3a9119242703c6f7f7c93daafc380ea96 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 7 Apr 2026 22:23:48 +0000 Subject: [PATCH 41/43] add debug log Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index d2e4fe1021ad..95fa9806a1da 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -155,5 +155,6 @@ async def transfer_run_periodically( # finish copying model_state.expert_buffer into # model_state.model.expert_weights[layer_idx] consumed_event.wait(stream=cuda_stream) + logger.debug("Layer %d transfer complete", layer_idx) assert model_state.pending_result is None layer_idx += 1 From cd99b1cb8e33c427c0630d044aef766f22066eaf Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 9 Apr 2026 21:06:15 +0000 Subject: [PATCH 42/43] comment updates Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 2 +- vllm/distributed/eplb/rebalance_execute.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 95fa9806a1da..5a0aa453929d 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -134,7 +134,7 @@ async def transfer_run_periodically( ) # Wait until all writes to expert_buffer have finished before making the - # AsynEplbLayerResult visible to the main thread. + # AsyncEplbLayerResult visible to the main thread. cuda_stream.synchronize() # This event guarantees that expert_buffer will not be overwritten by diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index d85ad075eced..32ebe2956646 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -48,7 +48,7 @@ class AsyncEplbLayerResult: """Index of the MoE layer that was transferred.""" new_physical_to_logical_map: torch.Tensor """ - New physical→logical mapping for all layers, on CPU. + New physical→logical mapping for layers_idx, on CPU. Shape: (num_physical_experts) """ is_unchanged: np.ndarray From 073651529300f3dd3598a8a23715fce881c7c5c2 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 16 Apr 2026 15:55:05 +0000 Subject: [PATCH 43/43] review comments Signed-off-by: Sage Moore --- vllm/distributed/eplb/async_worker.py | 4 ++-- vllm/distributed/eplb/eplb_state.py | 15 ++++++++++++--- vllm/distributed/eplb/eplb_utils.py | 10 +++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index 5a0aa453929d..5a1c6a1dfb1c 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -98,7 +98,7 @@ async def transfer_run_periodically( layer_idx = 0 # Set the async worker's CUDA stream on the communicator model_state.communicator.set_stream(cuda_stream) - current_num_layers = model_state.model.num_moe_layers + num_layers = model_state.model.num_moe_layers # Snapshot the physical_to_logical_map (synchronized with # rearrange_event) and copy it to CPU @@ -117,7 +117,7 @@ async def transfer_run_periodically( # of this loop will copy the new set of expert weights into # model_state.expert_buffer, which will be consumed by the main thread in # move_to_workspace - while model_state.rebalanced and layer_idx < current_num_layers: + while model_state.rebalanced and layer_idx < num_layers: ( is_unchanged, is_received_locally, diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 47e2b1902c23..a5722c64335d 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -176,7 +176,13 @@ class EplbModelState: """ rebalanced: bool """ - The flag indicates whether the experts rebalance have been computed. + This flag is only used when running Async EPLB. It is set to True by the main thread + after the new expert maps have been computed. This indicates that the async worker + should start transferring weights. move_to_workspace sets this flag to False when + all weights have been transferred and the new map has been successfully committed. + + rebalanced relies on the GIL to synchronize access between the main thread and + the async worker. """ eplb_stats: EplbStats | None """ @@ -195,6 +201,9 @@ class EplbModelState: Set by the async worker after all writes to expert_buffer are done. Consumed and reset to None by the main thread in move_to_workspace() after the contents of expert_buffer have been transferred out. At most one result is pending at a time. + + pending_result relies on the GIL to synchronize access between the main thread and + the async worker. """ @@ -570,8 +579,8 @@ def step( # Run _move_to_workspace if all ranks have finished transferring the # new weights to the intermediate buffer for eplb_model_state in self.model_states.values(): - # rebalanced must remain consistent amongst all ranks otherwise this - # all_reduce will hang + # rebalanced must remain consistent amongst all ranks otherwise the + # all_reduce in _all_ranks_result_ready will hang if eplb_model_state.rebalanced and self._all_ranks_result_ready( eplb_model_state ): diff --git a/vllm/distributed/eplb/eplb_utils.py b/vllm/distributed/eplb/eplb_utils.py index 886d9cc48442..92fffd229771 100644 --- a/vllm/distributed/eplb/eplb_utils.py +++ b/vllm/distributed/eplb/eplb_utils.py @@ -18,6 +18,10 @@ class CpuGpuEvent: Combines a CUDA event with a CPU threading event to enforce record->wait ordering across two threads. + This class is designed for exactly two threads: one producer that calls + record() and one consumer that calls wait(). Using it with more than two + threads is not supported and will produce undefined behavior. + CUDA events alone are insufficient for cross-thread synchronization because waiting on an unrecorded CUDA event is a no-op. The wait will return immediately instead of blocking. This class adds a threading.Event so @@ -47,7 +51,11 @@ def record(self, stream: torch.cuda.Stream | None = None): Should only be called by the main thread. """ - assert not self._recorded.is_set() + if self._recorded.is_set(): + raise RuntimeError( + "CpuGpuEvent.record() called before the previous event was " + "consumed by wait()" + ) self._event = torch.cuda.Event() self._event.record(stream) self._recorded.set()