From 41b9f4eab9e5aaf27536b8572e776c3fe4808814 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 27 Jul 2026 14:51:46 -0700 Subject: [PATCH 1/4] feat(trtllm): select conversation affinity rank owner Signed-off-by: krishung5 --- components/src/dynamo/trtllm/backend_args.py | 17 ++++- .../dynamo/trtllm/conversation_affinity.py | 15 +++-- .../trtllm/request_handlers/handler_base.py | 31 ++++++++-- .../trtllm/tests/test_trtllm_handler_base.py | 62 ++++++++++++++++++- .../dynamo/trtllm/tests/test_trtllm_unit.py | 34 ++++++++++ components/src/dynamo/trtllm/tests/utils.py | 1 + .../src/dynamo/trtllm/workers/llm_worker.py | 3 + 7 files changed, 149 insertions(+), 14 deletions(-) diff --git a/components/src/dynamo/trtllm/backend_args.py b/components/src/dynamo/trtllm/backend_args.py index b0fda5910348..2fd3678db239 100644 --- a/components/src/dynamo/trtllm/backend_args.py +++ b/components/src/dynamo/trtllm/backend_args.py @@ -101,8 +101,20 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: flag_name="--conversation-affinity", env_var="DYN_ENGINE_CONV_AFFINITY", default=False, - help="Force engine-owned conversation-affinity ADP routing: the engine picks the " - "attention-DP rank from the conversation id, even if the router selects a rank.", + help="Force TensorRT-LLM conversation-affinity ADP routing regardless of engine " + "config detection. Initial DP-rank placement is controlled by " + "--conversation-affinity-dp-rank-source.", + ) + add_argument( + g, + flag_name="--conversation-affinity-dp-rank-source", + env_var="DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + default="engine", + choices=["engine", "dynamo"], + help="Select initial attention-DP placement while conversation affinity is enabled. " + "'engine' lets TensorRT-LLM load-balance the first request; 'dynamo' forwards the " + "Dynamo router's selected rank and requires a TensorRT-LLM build containing " + "NVIDIA/TensorRT-LLM#16815 or equivalent.", ) add_argument( g, @@ -485,6 +497,7 @@ class DynamoTrtllmConfig(ConfigBase): expert_parallel_size: Optional[int] enable_attention_dp: bool conversation_affinity: bool + conversation_affinity_dp_rank_source: str kv_block_size: int gpus_per_node: Optional[int] = None max_batch_size: int diff --git a/components/src/dynamo/trtllm/conversation_affinity.py b/components/src/dynamo/trtllm/conversation_affinity.py index 0b28eadb52f0..e4a7c96d1f89 100644 --- a/components/src/dynamo/trtllm/conversation_affinity.py +++ b/components/src/dynamo/trtllm/conversation_affinity.py @@ -1,15 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Helpers for engine-owned conversation-aware ADP routing (opt-in). +"""Helpers for conversation-aware ADP routing (opt-in). When TensorRT-LLM's ``attention_dp_config.kv_cache_routing_conversation_affinity`` is enabled, the engine's ``ConversationAwareADPRouter`` pins a conversation to an -attention-DP rank (sticky, load-balanced first turn). Dynamo must then: +attention-DP rank. Dynamo must always forward the stable conversation id via +``ConversationParams``. Initial placement can then be owned by either: - 1. forward the stable conversation id via ``ConversationParams``, and - 2. NOT force ``attention_dp_rank`` — an explicit rank is honored *before* affinity - in the engine and would bypass it. + 1. TensorRT-LLM, which load-balances the first turn when no explicit rank is sent; or + 2. Dynamo, which sends its selected ``attention_dp_rank`` and relies on TensorRT-LLM + to record that initial ``conversation_id -> rank`` binding. + +Dynamo-owned initial placement requires a TensorRT-LLM build containing +NVIDIA/TensorRT-LLM#16815 (or equivalent). The default remains engine-owned so older +TensorRT-LLM builds retain their existing behavior. The conversation id is the frontend-forwarded ``agent_context.session_id`` — no new routing plumbing is added on the Rust side (see the serialized diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index dd645a100e83..9b72a5b4968f 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -249,8 +249,10 @@ class RequestHandlerConfig: additional_metrics: Optional["AdditionalMetricsCollector"] = None max_seq_len: Optional[int] = None disagg_machine_id: int = 0 # 10-bit machine_id for snowflake disagg_request_id - # Force engine-owned conversation-affinity ADP routing regardless of engine detection. + # Force conversation-affinity ADP routing regardless of engine config detection. conversation_affinity: bool = False + # Select whether the engine or Dynamo owns initial DP-rank placement in affinity mode. + conversation_affinity_dp_rank_source: str = "engine" class HandlerBase(BaseGenerativeHandler): @@ -278,6 +280,9 @@ def __init__(self, config: RequestHandlerConfig): # Manual override (--conversation-affinity / DYN_ENGINE_CONV_AFFINITY) to force # engine-side assignment of conversation-affinity regardless of engine detection. self._engine_conversation_affinity_override: bool = config.conversation_affinity + self._conversation_affinity_dp_rank_source: str = ( + config.conversation_affinity_dp_rank_source + ) self.encode_client = config.encode_client self.multimodal_processor = config.multimodal_processor self.first_generation = True @@ -1115,9 +1120,8 @@ async def _generate_locally_impl( # Build trace headers for distributed tracing trace_headers = context.trace_headers() - # Resolve the engine-owned conversation-affinity gate once (lazily; the engine is - # initialized by first request). When on, the engine's ConversationAwareADPRouter - # picks the attention-DP rank from the conversation id, so we must NOT force a rank. + # Resolve the engine conversation-affinity gate once (lazily; the engine is + # initialized by first request). if self._conversation_affinity is None: if ( self._engine_conversation_affinity_override @@ -1146,11 +1150,26 @@ async def _generate_locally_impl( conversation_params = None scheduling_params = None if conv_affinity: - # Let the engine pick the rank from the conversation id (agent_context.session_id); - # do NOT force a rank (an explicit rank is honored before affinity and bypasses it). conversation_params = conversation_params_for( session_id_from_request(request) ) + if ( + self._conversation_affinity_dp_rank_source == "dynamo" + and conversation_params is not None + and dp_rank is not None + ): + # TensorRT-LLM#16815 records this explicit first-turn placement as the + # conversation binding. On later turns, the recorded binding takes + # precedence if Dynamo supplies a different rank. + scheduling_params = SchedulingParams( + attention_dp_rank=dp_rank, + attention_dp_relax=False, + ) + logging.debug( + "Using dynamo router dp_rank=%s for initial conversation-affinity " + "placement", + dp_rank, + ) elif dp_rank is not None: scheduling_params = SchedulingParams( attention_dp_rank=dp_rank, diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index d7cca4b36c3e..d6d17acf5cc6 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -1069,11 +1069,17 @@ class TestConversationAffinity: ``SchedulingParams`` as before. """ - def _make_handler(self, *, conversation_affinity: bool) -> HandlerBase: + def _make_handler( + self, + *, + conversation_affinity: bool, + dp_rank_source: str = "engine", + ) -> HandlerBase: config = MagicMock() config.shutdown_event = None config.disaggregation_mode = DisaggregationMode.AGGREGATED config.conversation_affinity = False + config.conversation_affinity_dp_rank_source = dp_rank_source handler = _ConcreteHandler(config) handler.publisher = None handler.multimodal_processor = None @@ -1182,6 +1188,60 @@ async def test_affinity_on_with_session_id_suppresses_rank(self, monkeypatch): assert conv_params is not None assert conv_params.conversation_id == "run-42:agent-0" + @pytest.mark.asyncio + async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypatch): + """Dynamo-owned placement forwards both the rank and conversation id.""" + monkeypatch.setattr( + "dynamo.trtllm.conversation_affinity.ConversationParams", + _FakeConversationParams, + ) + handler = self._make_handler( + conversation_affinity=True, + dp_rank_source="dynamo", + ) + kwargs = await self._drive( + handler, + { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"dp_rank": 3}, + "agent_context": {"session_id": "run-42:agent-0"}, + }, + ) + scheduling_params = kwargs["scheduling_params"] + assert scheduling_params is not None + assert scheduling_params.attention_dp_rank == 3 + assert scheduling_params.attention_dp_relax is False + conv_params = kwargs["conversation_params"] + assert conv_params is not None + assert conv_params.conversation_id == "run-42:agent-0" + + @pytest.mark.asyncio + async def test_affinity_on_with_dynamo_rank_source_and_no_session_suppresses_rank( + self, monkeypatch + ): + """Without a conversation id, preserve TRT-LLM's no-id balancing path.""" + monkeypatch.setattr( + "dynamo.trtllm.conversation_affinity.ConversationParams", + _FakeConversationParams, + ) + handler = self._make_handler( + conversation_affinity=True, + dp_rank_source="dynamo", + ) + kwargs = await self._drive( + handler, + { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"dp_rank": 3}, + }, + ) + assert kwargs["scheduling_params"] is None + assert kwargs["conversation_params"] is None + @pytest.mark.asyncio async def test_affinity_on_without_session_id_passes_none_conversation_params( self, monkeypatch diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py index 7471eb3355f3..86b59a3195d7 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py @@ -313,6 +313,40 @@ def test_conversation_affinity_defaults_false(monkeypatch): assert config.conversation_affinity is False +def test_conversation_affinity_dp_rank_source_cli_flag(monkeypatch): + """The initial affinity placement owner can be selected on the CLI.""" + monkeypatch.delenv( + "DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + raising=False, + ) + config = parse_args( + [ + "--model", + "fake-model", + "--conversation-affinity-dp-rank-source", + "dynamo", + ] + ) + assert config.conversation_affinity_dp_rank_source == "dynamo" + + +def test_conversation_affinity_dp_rank_source_env_var(monkeypatch): + """DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE selects initial placement.""" + monkeypatch.setenv("DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", "dynamo") + config = parse_args(["--model", "fake-model"]) + assert config.conversation_affinity_dp_rank_source == "dynamo" + + +def test_conversation_affinity_dp_rank_source_defaults_engine(monkeypatch): + """TRT-LLM keeps ownership of first-turn placement by default.""" + monkeypatch.delenv( + "DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + raising=False, + ) + config = parse_args(["--model", "fake-model"]) + assert config.conversation_affinity_dp_rank_source == "engine" + + def test_enable_multimodal_rejects_diffusion_modality(): with pytest.raises(ValueError, match="--enable-multimodal cannot be combined"): parse_args( diff --git a/components/src/dynamo/trtllm/tests/utils.py b/components/src/dynamo/trtllm/tests/utils.py index a9157020370a..aa20622a3fde 100644 --- a/components/src/dynamo/trtllm/tests/utils.py +++ b/components/src/dynamo/trtllm/tests/utils.py @@ -33,5 +33,6 @@ def create_mock_request_handler_config( config.kv_block_size = 32 config.shutdown_event = None config.conversation_affinity = False + config.conversation_affinity_dp_rank_source = "engine" config.encoder_cache_capacity_gb = encoder_cache_capacity_gb return config diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index 6dd9445db061..523ba06f3499 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -777,6 +777,9 @@ async def init_llm_worker( max_seq_len=config.max_seq_len, disagg_machine_id=int(endpoint.connection_id()) % 1021, conversation_affinity=config.conversation_affinity, + conversation_affinity_dp_rank_source=( + config.conversation_affinity_dp_rank_source + ), ) media_decoder = None From b1c4d2d6cc7fc76d1890033e7f4c63671270550b Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 28 Jul 2026 09:40:06 -0700 Subject: [PATCH 2/4] fix(trtllm): fall back when explicit affinity is unsupported Signed-off-by: krishung5 --- .../dynamo/trtllm/conversation_affinity.py | 7 ++++ .../trtllm/request_handlers/handler_base.py | 11 +++++ .../trtllm/tests/test_trtllm_handler_base.py | 41 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/components/src/dynamo/trtllm/conversation_affinity.py b/components/src/dynamo/trtllm/conversation_affinity.py index e4a7c96d1f89..eda6ba00f4c4 100644 --- a/components/src/dynamo/trtllm/conversation_affinity.py +++ b/components/src/dynamo/trtllm/conversation_affinity.py @@ -36,6 +36,13 @@ except ImportError: # pragma: no cover - depends on installed wheel ConversationParams = None # type: ignore[assignment] +try: + from tensorrt_llm._torch.pyexecutor.scheduler.adp_router import ( + SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING, + ) +except ImportError: # pragma: no cover - depends on installed wheel + SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING = False + CONVERSATION_PARAMS_AVAILABLE: bool = ConversationParams is not None diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 9b72a5b4968f..6dd77487b7a1 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -48,6 +48,7 @@ from dynamo.trtllm.constants import DisaggregationMode from dynamo.trtllm.conversation_affinity import ( CONVERSATION_PARAMS_AVAILABLE, + SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING, conversation_params_for, engine_conversation_affinity_enabled, session_id_from_request, @@ -1143,6 +1144,16 @@ async def _generate_locally_impl( "release newer than 1.3.0rc20)." ) conv_affinity = self._conversation_affinity + if ( + conv_affinity + and self._conversation_affinity_dp_rank_source == "dynamo" + and not SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING + ): + logger.warning( + "Dynamo-owned conversation-affinity DP-rank placement is unsupported by " + "this TensorRT-LLM build; falling back to engine-owned initial placement." + ) + self._conversation_affinity_dp_rank_source = "engine" # Extract dp_rank from request's routing hints for attention DP routing routing = request.get("routing", {}) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index d6d17acf5cc6..beed4862ef55 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import logging import re as re_mod from copy import deepcopy from dataclasses import dataclass @@ -1195,6 +1196,11 @@ async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypat "dynamo.trtllm.conversation_affinity.ConversationParams", _FakeConversationParams, ) + monkeypatch.setattr( + "dynamo.trtllm.request_handlers.handler_base." + "SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING", + True, + ) handler = self._make_handler( conversation_affinity=True, dp_rank_source="dynamo", @@ -1217,6 +1223,41 @@ async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypat assert conv_params is not None assert conv_params.conversation_id == "run-42:agent-0" + @pytest.mark.asyncio + async def test_unsupported_dynamo_rank_source_warns_and_falls_back( + self, monkeypatch, caplog + ): + """An older engine safely falls back instead of losing later-turn affinity.""" + monkeypatch.setattr( + "dynamo.trtllm.conversation_affinity.ConversationParams", + _FakeConversationParams, + ) + monkeypatch.setattr( + "dynamo.trtllm.request_handlers.handler_base." + "SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING", + False, + ) + handler = self._make_handler( + conversation_affinity=True, + dp_rank_source="dynamo", + ) + with caplog.at_level(logging.WARNING): + kwargs = await self._drive( + handler, + { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"dp_rank": 3}, + "agent_context": {"session_id": "run-42:agent-0"}, + }, + ) + + assert kwargs["scheduling_params"] is None + assert kwargs["conversation_params"].conversation_id == "run-42:agent-0" + assert handler._conversation_affinity_dp_rank_source == "engine" + assert "falling back to engine-owned initial placement" in caplog.text + @pytest.mark.asyncio async def test_affinity_on_with_dynamo_rank_source_and_no_session_suppresses_rank( self, monkeypatch From 5a8ff5333577eb70289a12a930320ec5293a5744 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 28 Jul 2026 09:49:29 -0700 Subject: [PATCH 3/4] revert(trtllm): remove explicit binding capability gate Signed-off-by: krishung5 --- .../dynamo/trtllm/conversation_affinity.py | 7 ---- .../trtllm/request_handlers/handler_base.py | 11 ----- .../trtllm/tests/test_trtllm_handler_base.py | 41 ------------------- 3 files changed, 59 deletions(-) diff --git a/components/src/dynamo/trtllm/conversation_affinity.py b/components/src/dynamo/trtllm/conversation_affinity.py index eda6ba00f4c4..e4a7c96d1f89 100644 --- a/components/src/dynamo/trtllm/conversation_affinity.py +++ b/components/src/dynamo/trtllm/conversation_affinity.py @@ -36,13 +36,6 @@ except ImportError: # pragma: no cover - depends on installed wheel ConversationParams = None # type: ignore[assignment] -try: - from tensorrt_llm._torch.pyexecutor.scheduler.adp_router import ( - SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING, - ) -except ImportError: # pragma: no cover - depends on installed wheel - SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING = False - CONVERSATION_PARAMS_AVAILABLE: bool = ConversationParams is not None diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 6dd77487b7a1..9b72a5b4968f 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -48,7 +48,6 @@ from dynamo.trtllm.constants import DisaggregationMode from dynamo.trtllm.conversation_affinity import ( CONVERSATION_PARAMS_AVAILABLE, - SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING, conversation_params_for, engine_conversation_affinity_enabled, session_id_from_request, @@ -1144,16 +1143,6 @@ async def _generate_locally_impl( "release newer than 1.3.0rc20)." ) conv_affinity = self._conversation_affinity - if ( - conv_affinity - and self._conversation_affinity_dp_rank_source == "dynamo" - and not SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING - ): - logger.warning( - "Dynamo-owned conversation-affinity DP-rank placement is unsupported by " - "this TensorRT-LLM build; falling back to engine-owned initial placement." - ) - self._conversation_affinity_dp_rank_source = "engine" # Extract dp_rank from request's routing hints for attention DP routing routing = request.get("routing", {}) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index beed4862ef55..d6d17acf5cc6 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio -import logging import re as re_mod from copy import deepcopy from dataclasses import dataclass @@ -1196,11 +1195,6 @@ async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypat "dynamo.trtllm.conversation_affinity.ConversationParams", _FakeConversationParams, ) - monkeypatch.setattr( - "dynamo.trtllm.request_handlers.handler_base." - "SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING", - True, - ) handler = self._make_handler( conversation_affinity=True, dp_rank_source="dynamo", @@ -1223,41 +1217,6 @@ async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypat assert conv_params is not None assert conv_params.conversation_id == "run-42:agent-0" - @pytest.mark.asyncio - async def test_unsupported_dynamo_rank_source_warns_and_falls_back( - self, monkeypatch, caplog - ): - """An older engine safely falls back instead of losing later-turn affinity.""" - monkeypatch.setattr( - "dynamo.trtllm.conversation_affinity.ConversationParams", - _FakeConversationParams, - ) - monkeypatch.setattr( - "dynamo.trtllm.request_handlers.handler_base." - "SUPPORTS_EXPLICIT_DP_RANK_CONVERSATION_BINDING", - False, - ) - handler = self._make_handler( - conversation_affinity=True, - dp_rank_source="dynamo", - ) - with caplog.at_level(logging.WARNING): - kwargs = await self._drive( - handler, - { - "token_ids": [1, 2, 3], - "stop_conditions": {"max_tokens": 10}, - "sampling_options": {"temperature": 0.7}, - "routing": {"dp_rank": 3}, - "agent_context": {"session_id": "run-42:agent-0"}, - }, - ) - - assert kwargs["scheduling_params"] is None - assert kwargs["conversation_params"].conversation_id == "run-42:agent-0" - assert handler._conversation_affinity_dp_rank_source == "engine" - assert "falling back to engine-owned initial placement" in caplog.text - @pytest.mark.asyncio async def test_affinity_on_with_dynamo_rank_source_and_no_session_suppresses_rank( self, monkeypatch From 234196dedacc4559e5d01122a9d60f354c772d98 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 28 Jul 2026 10:24:17 -0700 Subject: [PATCH 4/4] docs(trtllm): explain conversation-aware DP routing --- .../backends/trtllm/trtllm-dp-rank-routing.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md b/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md index 46718e96d350..a9ffd8962091 100644 --- a/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md +++ b/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md @@ -15,6 +15,44 @@ TensorRT-LLM supports [attention data parallelism](https://lmsys.org/blog/2024-1 - **Dynamo DP Rank Routing**: The router selects the optimal DP rank based on KV cache overlap and instructs TRT-LLM to use that rank with strict routing (`attention_dp_relax=False`). Use this with `--router-mode kv` for cache-aware routing. - **TRT-LLM Internal Routing**: TRT-LLM's scheduler assigns DP ranks internally. Use this with `--router-mode round-robin` or `random` when KV-aware routing isn't needed. +### Conversation-Aware DP Rank Routing + +To keep a conversation on the same worker and attention-DP rank, enable conversation affinity in both the [Dynamo router](../../components/router/router-configuration.md#session-affinity) and the [TensorRT-LLM ADP router](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md#rank-level-routing-within-a-context-server). Dynamo passes the stable conversation ID to TensorRT-LLM through `ConversationParams`. Use `--conversation-affinity-dp-rank-source` to choose which component owns the initial attention-DP rank placement: + +| Value | Initial placement | Later requests | +| --- | --- | --- | +| `engine` (default) | TensorRT-LLM selects the rank with its conversation-aware ADP router. | TensorRT-LLM reuses the stored conversation-to-rank binding. | +| `dynamo` | Dynamo forwards the KV router's selected rank as a strict `SchedulingParams.attention_dp_rank`. | TensorRT-LLM reuses the stored conversation-to-rank binding, even if a later Dynamo routing hint differs. | + +Enable conversation-aware routing in the TensorRT-LLM engine configuration: + +```yaml +enable_attention_dp: true +attention_dp_config: + kv_cache_routing_conversation_affinity: true +``` + +Then pass that configuration and configure the rank source on the Dynamo TensorRT-LLM worker: + +```bash +python3 -m dynamo.trtllm \ + --model-path \ + --extra-engine-args engine.yaml \ + --conversation-affinity \ + --conversation-affinity-dp-rank-source engine +``` + +You can set the same option with the `DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE` environment variable: + +```bash +export DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE=dynamo +``` + +> [!WARNING] +> The `dynamo` rank source requires a TensorRT-LLM version containing [NVIDIA/TensorRT-LLM#16815](https://github.com/NVIDIA/TensorRT-LLM/pull/16815) or equivalent support. Enable it only after Dynamo's TensorRT-LLM dependency includes that change. Without it, TensorRT-LLM may accept the explicit first-turn rank without recording it as the conversation binding, allowing later turns to move to another rank. + +The rank-source option changes only who selects the first rank. Both modes require a stable conversation ID, and TensorRT-LLM owns the conversation-to-rank binding after initial placement. + ### Enabling DP Rank Routing ```bash