From e83e196cc7cccbbfee4b147e51802dc021159a00 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 23 Jul 2026 15:11:21 -0700 Subject: [PATCH 1/3] [None][fix] bind explicit rank for new conversations Signed-off-by: krishung5 --- .../_torch/pyexecutor/scheduler/adp_router.py | 41 ++++++++++++++++++- .../_torch/executor/test_adp_router.py | 20 +++++++-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py index 21fd55971bd6..d19051d683c4 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py @@ -857,6 +857,42 @@ def _record_target_rank(self, conv_id: str, rank: int) -> None: while len(self._conv_to_rank) > self._max_sessions: self._conv_to_rank.popitem(last=False) + def _assign_new_conversation_explicit_dp_ranks( + self, + requests: List["RequestQueueItem"], + all_ranks_new_requests: Dict[int, List["RequestQueueItem"]], + all_ranks_num_active_requests: List[int], + max_num_active_requests: int, + ) -> List["RequestQueueItem"]: + """Place explicit first turns and establish their affinity binding. + + Once a conversation is bound, its recorded rank takes precedence over + later explicit rank hints. Requests whose explicit target is full remain + eligible for the normal affinity/load-balanced path below. + """ + remaining: List["RequestQueueItem"] = [] + for req_item in requests: + conv_id = self._conversation_id(req_item) + if conv_id is not None and conv_id in self._conv_to_rank: + remaining.append(req_item) + continue + + scheduling_params = getattr(req_item.request, "py_scheduling_params", None) + target_dp_rank = ( + scheduling_params.attention_dp_rank if scheduling_params is not None else None + ) + if ( + target_dp_rank is not None + and all_ranks_num_active_requests[target_dp_rank] < max_num_active_requests + ): + all_ranks_num_active_requests[target_dp_rank] += 1 + all_ranks_new_requests[target_dp_rank].append(req_item) + if conv_id is not None: + self._record_target_rank(conv_id, target_dp_rank) + else: + remaining.append(req_item) + return remaining + def route_requests( self, all_rank_states: list[RankState], @@ -877,8 +913,9 @@ def get_relax_value(req_item): sorted_requests = sorted(new_requests, key=get_relax_value) - # 1) Honour an explicit attention_dp_rank first (strict placement). - remaining_unscheduled = self._assign_explicit_dp_ranks( + # 1) Honour an explicit attention_dp_rank for a new conversation and + # record that placement. Existing conversations keep their binding. + remaining_unscheduled = self._assign_new_conversation_explicit_dp_ranks( sorted_requests, all_ranks_new_requests, all_ranks_num_active_requests, diff --git a/tests/unittest/_torch/executor/test_adp_router.py b/tests/unittest/_torch/executor/test_adp_router.py index 3d2c418beec8..f59a3c3b233a 100644 --- a/tests/unittest/_torch/executor/test_adp_router.py +++ b/tests/unittest/_torch/executor/test_adp_router.py @@ -1207,11 +1207,23 @@ def test_lru_eviction_bounds_map(self): assert len(router._conv_to_rank) == 2 assert set(router._conv_to_rank) == {"c3", "c4"} - def test_explicit_target_dp_rank_respected(self): + def test_explicit_target_dp_rank_establishes_binding(self): router = self._router(tp_size=4) - item = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) - pos = self._route(router, self._states(4), [item]) - assert pos[1] == 2 + first = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) + assert self._route(router, self._states(4), [first])[1] == 2 + assert router._conv_to_rank["A"] == 2 + + second = _make_conv_request_item(2, "A") + assert self._route(router, self._states(4), [second])[2] == 2 + + def test_existing_binding_wins_over_later_explicit_target(self): + router = self._router(tp_size=4) + first = _make_conv_request_item(1, "A", target_dp_rank=2, attention_dp_relax=False) + self._route(router, self._states(4), [first]) + + second = _make_conv_request_item(2, "A", target_dp_rank=1, attention_dp_relax=False) + assert self._route(router, self._states(4), [second])[2] == 2 + assert router._conv_to_rank["A"] == 2 def test_sticky_overflow_keeps_mapping(self): """When the home rank is saturated at the HARD cap (max_num_active_requests), From 445a73f11aa1a0406e8c831dce4871d3d9bfb7e4 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 28 Jul 2026 09:40:06 -0700 Subject: [PATCH 2/3] [None][fix] expose explicit rank binding capability Signed-off-by: krishung5 --- tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py index d19051d683c4..424ac8ce4002 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py @@ -35,6 +35,11 @@ HeapVal = namedtuple("HeapVal", ["num_tokens", "num_requests", "rank", "request_list"]) +# Capability consumed by external orchestrators that optionally select the +# initial DP rank. Builds without this marker may honor an explicit first-turn +# rank without recording it as the conversation's persistent affinity binding. +SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING = True + def _num_input_tokens(request) -> int: """Token count via the cheap num_input_tokens accessor (avoids materializing From a7c1ebe58d895dbc63d6b429b4719c1cccc4306b Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 28 Jul 2026 09:49:26 -0700 Subject: [PATCH 3/3] [None][revert] remove explicit rank capability marker Signed-off-by: krishung5 --- tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py index 424ac8ce4002..d19051d683c4 100644 --- a/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py +++ b/tensorrt_llm/_torch/pyexecutor/scheduler/adp_router.py @@ -35,11 +35,6 @@ HeapVal = namedtuple("HeapVal", ["num_tokens", "num_requests", "rank", "request_list"]) -# Capability consumed by external orchestrators that optionally select the -# initial DP rank. Builds without this marker may honor an explicit first-turn -# rank without recording it as the conversation's persistent affinity binding. -SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING = True - def _num_input_tokens(request) -> int: """Token count via the cheap num_input_tokens accessor (avoids materializing