diff --git a/components/src/dynamo/trtllm/backend_args.py b/components/src/dynamo/trtllm/backend_args.py index b9330268b32b..aba33f7eeb98 100644 --- a/components/src/dynamo/trtllm/backend_args.py +++ b/components/src/dynamo/trtllm/backend_args.py @@ -90,8 +90,20 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: flag_name="--conversation-affinity", env_var="DYN_ENGINE_CONV_AFFINITY", default=False, - help="Force engine-owned conversation-affinity ADP routing: the engine picks the " - "attention-DP rank from the conversation id, even if the router selects a rank.", + help="Force TensorRT-LLM conversation-affinity ADP routing regardless of engine " + "config detection. Initial DP-rank placement is controlled by " + "--conversation-affinity-dp-rank-source.", + ) + add_argument( + g, + flag_name="--conversation-affinity-dp-rank-source", + env_var="DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + default="engine", + choices=["engine", "dynamo"], + help="Select initial attention-DP placement while conversation affinity is enabled. " + "'engine' lets TensorRT-LLM load-balance the first request; 'dynamo' forwards the " + "Dynamo router's selected rank and requires a TensorRT-LLM build containing " + "NVIDIA/TensorRT-LLM#16815 or equivalent.", ) add_argument( g, @@ -477,6 +489,7 @@ class DynamoTrtllmConfig(ConfigBase): expert_parallel_size: Optional[int] enable_attention_dp: bool conversation_affinity: bool + conversation_affinity_dp_rank_source: str kv_block_size: int gpus_per_node: Optional[int] = None max_batch_size: int diff --git a/components/src/dynamo/trtllm/conversation_affinity.py b/components/src/dynamo/trtllm/conversation_affinity.py index 0b28eadb52f0..e4a7c96d1f89 100644 --- a/components/src/dynamo/trtllm/conversation_affinity.py +++ b/components/src/dynamo/trtllm/conversation_affinity.py @@ -1,15 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Helpers for engine-owned conversation-aware ADP routing (opt-in). +"""Helpers for conversation-aware ADP routing (opt-in). When TensorRT-LLM's ``attention_dp_config.kv_cache_routing_conversation_affinity`` is enabled, the engine's ``ConversationAwareADPRouter`` pins a conversation to an -attention-DP rank (sticky, load-balanced first turn). Dynamo must then: +attention-DP rank. Dynamo must always forward the stable conversation id via +``ConversationParams``. Initial placement can then be owned by either: - 1. forward the stable conversation id via ``ConversationParams``, and - 2. NOT force ``attention_dp_rank`` — an explicit rank is honored *before* affinity - in the engine and would bypass it. + 1. TensorRT-LLM, which load-balances the first turn when no explicit rank is sent; or + 2. Dynamo, which sends its selected ``attention_dp_rank`` and relies on TensorRT-LLM + to record that initial ``conversation_id -> rank`` binding. + +Dynamo-owned initial placement requires a TensorRT-LLM build containing +NVIDIA/TensorRT-LLM#16815 (or equivalent). The default remains engine-owned so older +TensorRT-LLM builds retain their existing behavior. The conversation id is the frontend-forwarded ``agent_context.session_id`` — no new routing plumbing is added on the Rust side (see the serialized diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 784ee346bca5..e9e17b08aa15 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -251,8 +251,10 @@ class RequestHandlerConfig: additional_metrics: Optional["AdditionalMetricsCollector"] = None max_seq_len: Optional[int] = None disagg_machine_id: int = 0 # 10-bit machine_id for snowflake disagg_request_id - # Force engine-owned conversation-affinity ADP routing regardless of engine detection. + # Force conversation-affinity ADP routing regardless of engine config detection. conversation_affinity: bool = False + # Select whether the engine or Dynamo owns initial DP-rank placement in affinity mode. + conversation_affinity_dp_rank_source: str = "engine" class HandlerBase(BaseGenerativeHandler): @@ -280,6 +282,9 @@ def __init__(self, config: RequestHandlerConfig): # Manual override (--conversation-affinity / DYN_ENGINE_CONV_AFFINITY) to force # engine-side assignment of conversation-affinity regardless of engine detection. self._engine_conversation_affinity_override: bool = config.conversation_affinity + self._conversation_affinity_dp_rank_source: str = ( + config.conversation_affinity_dp_rank_source + ) self.encode_client = config.encode_client self.multimodal_processor = config.multimodal_processor self.first_generation = True @@ -1151,9 +1156,8 @@ async def _generate_locally_impl( # Build trace headers for distributed tracing trace_headers = context.trace_headers() - # Resolve the engine-owned conversation-affinity gate once (lazily; the engine is - # initialized by first request). When on, the engine's ConversationAwareADPRouter - # picks the attention-DP rank from the conversation id, so we must NOT force a rank. + # Resolve the engine conversation-affinity gate once (lazily; the engine is + # initialized by first request). if self._conversation_affinity is None: if ( self._engine_conversation_affinity_override @@ -1182,11 +1186,26 @@ async def _generate_locally_impl( conversation_params = None scheduling_params = None if conv_affinity: - # Let the engine pick the rank from the conversation id (agent_context.session_id); - # do NOT force a rank (an explicit rank is honored before affinity and bypasses it). conversation_params = conversation_params_for( session_id_from_request(request) ) + if ( + self._conversation_affinity_dp_rank_source == "dynamo" + and conversation_params is not None + and dp_rank is not None + ): + # TensorRT-LLM#16815 records this explicit first-turn placement as the + # conversation binding. On later turns, the recorded binding takes + # precedence if Dynamo supplies a different rank. + scheduling_params = SchedulingParams( + attention_dp_rank=dp_rank, + attention_dp_relax=False, + ) + logging.debug( + "Using dynamo router dp_rank=%s for initial conversation-affinity " + "placement", + dp_rank, + ) elif dp_rank is not None: scheduling_params = SchedulingParams( attention_dp_rank=dp_rank, diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index 62a93be6710f..4be534b8fac7 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -1091,11 +1091,17 @@ class TestConversationAffinity: ``SchedulingParams`` as before. """ - def _make_handler(self, *, conversation_affinity: bool) -> HandlerBase: + def _make_handler( + self, + *, + conversation_affinity: bool, + dp_rank_source: str = "engine", + ) -> HandlerBase: config = MagicMock() config.shutdown_event = None config.disaggregation_mode = DisaggregationMode.AGGREGATED config.conversation_affinity = False + config.conversation_affinity_dp_rank_source = dp_rank_source handler = _ConcreteHandler(config) handler.publisher = None handler.multimodal_processor = None @@ -1204,6 +1210,60 @@ async def test_affinity_on_with_session_id_suppresses_rank(self, monkeypatch): assert conv_params is not None assert conv_params.conversation_id == "run-42:agent-0" + @pytest.mark.asyncio + async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypatch): + """Dynamo-owned placement forwards both the rank and conversation id.""" + monkeypatch.setattr( + "dynamo.trtllm.conversation_affinity.ConversationParams", + _FakeConversationParams, + ) + handler = self._make_handler( + conversation_affinity=True, + dp_rank_source="dynamo", + ) + kwargs = await self._drive( + handler, + { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"dp_rank": 3}, + "agent_context": {"session_id": "run-42:agent-0"}, + }, + ) + scheduling_params = kwargs["scheduling_params"] + assert scheduling_params is not None + assert scheduling_params.attention_dp_rank == 3 + assert scheduling_params.attention_dp_relax is False + conv_params = kwargs["conversation_params"] + assert conv_params is not None + assert conv_params.conversation_id == "run-42:agent-0" + + @pytest.mark.asyncio + async def test_affinity_on_with_dynamo_rank_source_and_no_session_suppresses_rank( + self, monkeypatch + ): + """Without a conversation id, preserve TRT-LLM's no-id balancing path.""" + monkeypatch.setattr( + "dynamo.trtllm.conversation_affinity.ConversationParams", + _FakeConversationParams, + ) + handler = self._make_handler( + conversation_affinity=True, + dp_rank_source="dynamo", + ) + kwargs = await self._drive( + handler, + { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"dp_rank": 3}, + }, + ) + assert kwargs["scheduling_params"] is None + assert kwargs["conversation_params"] is None + @pytest.mark.asyncio async def test_affinity_on_without_session_id_passes_none_conversation_params( self, monkeypatch diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py index 691c00fd3e30..bbfbb79bbc57 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_unit.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_unit.py @@ -318,6 +318,40 @@ def test_conversation_affinity_defaults_false(monkeypatch): assert config.conversation_affinity is False +def test_conversation_affinity_dp_rank_source_cli_flag(monkeypatch): + """The initial affinity placement owner can be selected on the CLI.""" + monkeypatch.delenv( + "DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + raising=False, + ) + config = parse_args( + [ + "--model", + "fake-model", + "--conversation-affinity-dp-rank-source", + "dynamo", + ] + ) + assert config.conversation_affinity_dp_rank_source == "dynamo" + + +def test_conversation_affinity_dp_rank_source_env_var(monkeypatch): + """DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE selects initial placement.""" + monkeypatch.setenv("DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", "dynamo") + config = parse_args(["--model", "fake-model"]) + assert config.conversation_affinity_dp_rank_source == "dynamo" + + +def test_conversation_affinity_dp_rank_source_defaults_engine(monkeypatch): + """TRT-LLM keeps ownership of first-turn placement by default.""" + monkeypatch.delenv( + "DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", + raising=False, + ) + config = parse_args(["--model", "fake-model"]) + assert config.conversation_affinity_dp_rank_source == "engine" + + def test_enable_multimodal_rejects_diffusion_modality(): with pytest.raises(ValueError, match="--enable-multimodal cannot be combined"): parse_args( diff --git a/components/src/dynamo/trtllm/tests/utils.py b/components/src/dynamo/trtllm/tests/utils.py index a9157020370a..aa20622a3fde 100644 --- a/components/src/dynamo/trtllm/tests/utils.py +++ b/components/src/dynamo/trtllm/tests/utils.py @@ -33,5 +33,6 @@ def create_mock_request_handler_config( config.kv_block_size = 32 config.shutdown_event = None config.conversation_affinity = False + config.conversation_affinity_dp_rank_source = "engine" config.encoder_cache_capacity_gb = encoder_cache_capacity_gb return config diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index 6dd9445db061..523ba06f3499 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -777,6 +777,9 @@ async def init_llm_worker( max_seq_len=config.max_seq_len, disagg_machine_id=int(endpoint.connection_id()) % 1021, conversation_affinity=config.conversation_affinity, + conversation_affinity_dp_rank_source=( + config.conversation_affinity_dp_rank_source + ), ) media_decoder = None diff --git a/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md b/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md index 46718e96d350..a9ffd8962091 100644 --- a/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md +++ b/docs/fern/backends/trtllm/trtllm-dp-rank-routing.md @@ -15,6 +15,44 @@ TensorRT-LLM supports [attention data parallelism](https://lmsys.org/blog/2024-1 - **Dynamo DP Rank Routing**: The router selects the optimal DP rank based on KV cache overlap and instructs TRT-LLM to use that rank with strict routing (`attention_dp_relax=False`). Use this with `--router-mode kv` for cache-aware routing. - **TRT-LLM Internal Routing**: TRT-LLM's scheduler assigns DP ranks internally. Use this with `--router-mode round-robin` or `random` when KV-aware routing isn't needed. +### Conversation-Aware DP Rank Routing + +To keep a conversation on the same worker and attention-DP rank, enable conversation affinity in both the [Dynamo router](../../components/router/router-configuration.md#session-affinity) and the [TensorRT-LLM ADP router](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md#rank-level-routing-within-a-context-server). Dynamo passes the stable conversation ID to TensorRT-LLM through `ConversationParams`. Use `--conversation-affinity-dp-rank-source` to choose which component owns the initial attention-DP rank placement: + +| Value | Initial placement | Later requests | +| --- | --- | --- | +| `engine` (default) | TensorRT-LLM selects the rank with its conversation-aware ADP router. | TensorRT-LLM reuses the stored conversation-to-rank binding. | +| `dynamo` | Dynamo forwards the KV router's selected rank as a strict `SchedulingParams.attention_dp_rank`. | TensorRT-LLM reuses the stored conversation-to-rank binding, even if a later Dynamo routing hint differs. | + +Enable conversation-aware routing in the TensorRT-LLM engine configuration: + +```yaml +enable_attention_dp: true +attention_dp_config: + kv_cache_routing_conversation_affinity: true +``` + +Then pass that configuration and configure the rank source on the Dynamo TensorRT-LLM worker: + +```bash +python3 -m dynamo.trtllm \ + --model-path \ + --extra-engine-args engine.yaml \ + --conversation-affinity \ + --conversation-affinity-dp-rank-source engine +``` + +You can set the same option with the `DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE` environment variable: + +```bash +export DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE=dynamo +``` + +> [!WARNING] +> The `dynamo` rank source requires a TensorRT-LLM version containing [NVIDIA/TensorRT-LLM#16815](https://github.com/NVIDIA/TensorRT-LLM/pull/16815) or equivalent support. Enable it only after Dynamo's TensorRT-LLM dependency includes that change. Without it, TensorRT-LLM may accept the explicit first-turn rank without recording it as the conversation binding, allowing later turns to move to another rank. + +The rank-source option changes only who selects the first rank. Both modes require a stable conversation ID, and TensorRT-LLM owns the conversation-to-rank binding after initial placement. + ### Enabling DP Rank Routing ```bash