Repository navigation
[EPLB] Refactor Async EPLB synchronization logic #37601
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
5dc4639
44f4f11
ace8304
094126b
e7e70de
f4fe482
4e5bf9a
4b3ac69
95a0c0c
2f7ed7f
914cf4e
13ac084
10845df
2e37b21
cf7c9c1
d5177e4
837f665
9a78b8e
544c9ff
bbde306
2e2dd8f
dff2428
81bb2dd
67aadc5
3f6b4ce
b526659
1cd8948
cea868a
26751de
d00793e
0610ad4
5d0f02e
bf7b154
a12732d
2d69c8f
e812dd8
d61149c
4a17c8f
0b64b6a
3545c13
f2b7647
894a1ef
2f1c851
99bf9c3
f2d3bcf
ae5709b
cd99b1c
7c38536
d5eca69
0736515
9993a24
5b106b6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,7 @@ | |
| from vllm.distributed.parallel_state import get_eplb_group | ||
| from vllm.logger import init_logger | ||
|
|
||
| from .rebalance_execute import transfer_layer | ||
| from .rebalance_execute import AsyncEPLBLayerResult, transfer_layer | ||
|
|
||
| if TYPE_CHECKING: | ||
| from .eplb_state import EplbModelState, EplbState | ||
|
|
@@ -60,12 +60,15 @@ def run_rebalance_experts( | |
| model_state: "EplbModelState", | ||
| eplb_state: "EplbState", | ||
| physical_to_logical_map_cpu: torch.Tensor, | ||
| ) -> None: | ||
| ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | ||
| """Compute new expert mappings for one rebalancing cycle. | ||
|
|
||
| Returns: new_physical_to_logical_map | ||
| """ | ||
| assert model_state.eplb_stats is not None | ||
| eplb_stats = model_state.eplb_stats | ||
|
|
||
| # Wait for the main thread's all-reduce and clone to complete before | ||
| # accessing the global_expert_load_window tensor. | ||
| # Wait for global_expert_load_window tensor to be ready. | ||
| assert model_state.window_ready_event is not None | ||
| model_state.window_ready_event.wait() | ||
| model_state.window_ready_event = None | ||
|
|
@@ -87,17 +90,20 @@ def run_rebalance_experts( | |
| ) | ||
| assert new_physical_to_logical_map.device == torch.device("cpu") | ||
|
|
||
| model_state.new_physical_to_logical_map = new_physical_to_logical_map | ||
|
|
||
| max_slots = model_state.logical_to_physical_map.shape[-1] | ||
| padded_logical = torch.nn.functional.pad( | ||
| new_logical_to_physical_map = torch.nn.functional.pad( | ||
| new_logical_to_physical_map, | ||
| (0, max(0, max_slots - new_logical_to_physical_map.shape[-1])), | ||
| value=-1, | ||
| ).to(model_state.logical_to_physical_map.device) | ||
| new_replica = new_logical_replica_count.to(model_state.logical_replica_count.device) | ||
| model_state.new_logical_to_physical_map = padded_logical | ||
| model_state.new_logical_replica_count = new_replica | ||
| new_logical_replica_count = new_logical_replica_count.to( | ||
| model_state.logical_replica_count.device | ||
| ) | ||
| return ( | ||
| new_physical_to_logical_map, | ||
| new_logical_to_physical_map, | ||
| new_logical_replica_count, | ||
| ) | ||
|
|
||
|
|
||
| async def transfer_run_periodically( | ||
|
|
@@ -112,77 +118,58 @@ async def transfer_run_periodically( | |
|
|
||
| assert state.is_async | ||
| for model_state in state.model_states.values(): | ||
| rebalancing_algorithm_executed = False | ||
| physical_to_logical_map_cpu = None | ||
| layer_idx = 0 | ||
| current_num_layers = model_state.model.num_moe_layers | ||
| while ( | ||
| model_state.rebalanced | ||
| and model_state.layer_to_transfer < current_num_layers | ||
| ): | ||
| if not model_state.ep_buffer_ready and model_state.rebalanced: | ||
| # Polling the lock directly in the async thread avoids | ||
| # the thread switch overhead of asyncio.to_thread. | ||
| # This is typically faster than offloading to a worker thread. | ||
| while not model_state.buffer_lock.acquire(blocking=False): | ||
| await asyncio.sleep(0) | ||
| try: | ||
| if model_state.layer_to_transfer >= current_num_layers: | ||
| break | ||
| if ( | ||
| not rebalancing_algorithm_executed | ||
| or model_state.new_physical_to_logical_map is None | ||
| ): | ||
| # Move the physical_to_logical_map to CPU | ||
| # for rebalancing and transfer_layer. | ||
| physical_to_logical_map_cpu = ( | ||
| model_state.physical_to_logical_map.cpu() | ||
| ) | ||
| run_rebalance_experts( | ||
| model_state, state, physical_to_logical_map_cpu | ||
| ) | ||
| rebalancing_algorithm_executed = True | ||
| logger.info( | ||
| "Async worker computed new indices for model %s", | ||
| model_state.model_name, | ||
| ) | ||
|
|
||
| assert model_state.new_physical_to_logical_map is not None | ||
| assert physical_to_logical_map_cpu is not None | ||
|
|
||
| layer_idx = model_state.layer_to_transfer | ||
| old_layer_indices = physical_to_logical_map_cpu[layer_idx] | ||
| new_layer_indices = model_state.new_physical_to_logical_map[ | ||
| layer_idx | ||
| ] | ||
|
|
||
| # Wait for the main thread to finish consuming the buffer | ||
| # before initiating an EPLB transfer on another layer. | ||
| if model_state.buffer_consumed_event is not None: | ||
| cuda_stream.wait_event(model_state.buffer_consumed_event) | ||
| model_state.buffer_consumed_event = None | ||
|
|
||
| ( | ||
| model_state.is_unchanged, | ||
| model_state.is_received_locally, | ||
| model_state.recv_metadata, | ||
| ) = await transfer_layer( | ||
| old_layer_indices=old_layer_indices, | ||
| new_layer_indices=new_layer_indices, | ||
| expert_weights=model_state.model.expert_weights[layer_idx], | ||
| expert_weights_buffer=model_state.expert_buffer, | ||
| ep_group=eplb_group, | ||
| is_profile=is_profile, | ||
| cuda_stream=cuda_stream, | ||
| ) | ||
| event = torch.cuda.Event(blocking=False) | ||
| cuda_stream.record_event(event) | ||
| model_state.buffer_ready_event = event | ||
| model_state.ep_buffer_ready = 1 | ||
| finally: | ||
| model_state.buffer_lock.release() | ||
| else: | ||
| if not model_state.rebalanced: | ||
| break | ||
| await asyncio.sleep(0.001) | ||
|
|
||
| # Snapshot the physical_to_logical_map and copy it to CPU | ||
|
SageMoore marked this conversation as resolved.
Resolved
|
||
| with model_state.map_lock: | ||
| physical_to_logical_map_cpu = model_state.physical_to_logical_map.cpu() | ||
|
|
||
| ( | ||
| new_physical_to_logical_map, | ||
| new_logical_to_physical_map, | ||
| new_logical_replica_count, | ||
| ) = run_rebalance_experts(model_state, state, physical_to_logical_map_cpu) | ||
| logger.info( | ||
| "Async worker computed new indices for model %s", | ||
| model_state.model_name, | ||
| ) | ||
|
|
||
| while model_state.rebalanced and layer_idx < current_num_layers: | ||
|
SageMoore marked this conversation as resolved.
Resolved
|
||
| ( | ||
| is_unchanged, | ||
| is_received_locally, | ||
| recv_metadata, | ||
| ) = await transfer_layer( | ||
| old_layer_indices=physical_to_logical_map_cpu[layer_idx], | ||
| new_layer_indices=new_physical_to_logical_map[layer_idx], | ||
| expert_weights=model_state.model.expert_weights[layer_idx], | ||
| expert_weights_buffer=model_state.expert_buffer, | ||
| ep_group=eplb_group, | ||
| is_profile=is_profile, | ||
| cuda_stream=cuda_stream, | ||
| ) | ||
|
|
||
| # Wait until all writes to expert_buffer have finished before making the | ||
| # AsynEPLBLayerResult visible to the main thread. | ||
| cuda_stream.synchronize() | ||
|
|
||
| # This event guarantees that expert_buffer will not be overwritten by | ||
| # subsequent iterations of this loop until the main thread has consumed | ||
| # it. Record is called by the main thread after move_from_buffer(). | ||
| consumed_event = torch.cuda.Event() | ||
| cuda_stream.wait_event(consumed_event) | ||
|
SageMoore marked this conversation as resolved.
Resolved
|
||
|
|
||
| model_state.pending_result = AsyncEPLBLayerResult( | ||
| layer_idx=layer_idx, | ||
| new_physical_to_logical_map=new_physical_to_logical_map, | ||
| new_logical_to_physical_map=new_logical_to_physical_map, | ||
| new_logical_replica_count=new_logical_replica_count, | ||
| is_unchanged=is_unchanged, | ||
| is_received_locally=is_received_locally, | ||
| recv_metadata=recv_metadata, | ||
| consumed_event=consumed_event, | ||
| ) | ||
|
Comment on lines
+145
to
+152
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @SageMoore as you pointed out to me, IMO we should lock
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is also true for |
||
| layer_idx += 1 | ||
|
|
||
| state.rearrange_event.clear() | ||
Uh oh!
There was an error while loading. Please reload this page.