Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions components/src/dynamo/trtllm/backend_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,20 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
flag_name="--conversation-affinity",
env_var="DYN_ENGINE_CONV_AFFINITY",
default=False,
help="Force engine-owned conversation-affinity ADP routing: the engine picks the "
"attention-DP rank from the conversation id, even if the router selects a rank.",
help="Force TensorRT-LLM conversation-affinity ADP routing regardless of engine "
"config detection. Initial DP-rank placement is controlled by "
"--conversation-affinity-dp-rank-source.",
)
add_argument(
g,
flag_name="--conversation-affinity-dp-rank-source",
env_var="DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE",
default="engine",
choices=["engine", "dynamo"],
help="Select initial attention-DP placement while conversation affinity is enabled. "
"'engine' lets TensorRT-LLM load-balance the first request; 'dynamo' forwards the "
"Dynamo router's selected rank and requires a TensorRT-LLM build containing "
"NVIDIA/TensorRT-LLM#16815 or equivalent.",
)
add_argument(
g,
Expand Down Expand Up @@ -477,6 +489,7 @@ class DynamoTrtllmConfig(ConfigBase):
expert_parallel_size: Optional[int]
enable_attention_dp: bool
conversation_affinity: bool
conversation_affinity_dp_rank_source: str
kv_block_size: int
gpus_per_node: Optional[int] = None
max_batch_size: int
Expand Down
15 changes: 10 additions & 5 deletions components/src/dynamo/trtllm/conversation_affinity.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Helpers for engine-owned conversation-aware ADP routing (opt-in).
"""Helpers for conversation-aware ADP routing (opt-in).

When TensorRT-LLM's ``attention_dp_config.kv_cache_routing_conversation_affinity`` is
enabled, the engine's ``ConversationAwareADPRouter`` pins a conversation to an
attention-DP rank (sticky, load-balanced first turn). Dynamo must then:
attention-DP rank. Dynamo must always forward the stable conversation id via
``ConversationParams``. Initial placement can then be owned by either:

1. forward the stable conversation id via ``ConversationParams``, and
2. NOT force ``attention_dp_rank`` — an explicit rank is honored *before* affinity
in the engine and would bypass it.
1. TensorRT-LLM, which load-balances the first turn when no explicit rank is sent; or
2. Dynamo, which sends its selected ``attention_dp_rank`` and relies on TensorRT-LLM
to record that initial ``conversation_id -> rank`` binding.

Dynamo-owned initial placement requires a TensorRT-LLM build containing
NVIDIA/TensorRT-LLM#16815 (or equivalent). The default remains engine-owned so older
TensorRT-LLM builds retain their existing behavior.

The conversation id is the frontend-forwarded ``agent_context.session_id`` — no new
routing plumbing is added on the Rust side (see the serialized
Expand Down
31 changes: 25 additions & 6 deletions components/src/dynamo/trtllm/request_handlers/handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,10 @@ class RequestHandlerConfig:
additional_metrics: Optional["AdditionalMetricsCollector"] = None
max_seq_len: Optional[int] = None
disagg_machine_id: int = 0 # 10-bit machine_id for snowflake disagg_request_id
# Force engine-owned conversation-affinity ADP routing regardless of engine detection.
# Force conversation-affinity ADP routing regardless of engine config detection.
conversation_affinity: bool = False
# Select whether the engine or Dynamo owns initial DP-rank placement in affinity mode.
conversation_affinity_dp_rank_source: str = "engine"


class HandlerBase(BaseGenerativeHandler):
Expand Down Expand Up @@ -280,6 +282,9 @@ def __init__(self, config: RequestHandlerConfig):
# Manual override (--conversation-affinity / DYN_ENGINE_CONV_AFFINITY) to force
# engine-side assignment of conversation-affinity regardless of engine detection.
self._engine_conversation_affinity_override: bool = config.conversation_affinity
self._conversation_affinity_dp_rank_source: str = (
config.conversation_affinity_dp_rank_source
)
self.encode_client = config.encode_client
self.multimodal_processor = config.multimodal_processor
self.first_generation = True
Expand Down Expand Up @@ -1151,9 +1156,8 @@ async def _generate_locally_impl(
# Build trace headers for distributed tracing
trace_headers = context.trace_headers()

# Resolve the engine-owned conversation-affinity gate once (lazily; the engine is
# initialized by first request). When on, the engine's ConversationAwareADPRouter
# picks the attention-DP rank from the conversation id, so we must NOT force a rank.
# Resolve the engine conversation-affinity gate once (lazily; the engine is
# initialized by first request).
if self._conversation_affinity is None:
if (
self._engine_conversation_affinity_override
Expand Down Expand Up @@ -1182,11 +1186,26 @@ async def _generate_locally_impl(
conversation_params = None
scheduling_params = None
if conv_affinity:
# Let the engine pick the rank from the conversation id (agent_context.session_id);
# do NOT force a rank (an explicit rank is honored before affinity and bypasses it).
conversation_params = conversation_params_for(
session_id_from_request(request)
)
if (
self._conversation_affinity_dp_rank_source == "dynamo"
and conversation_params is not None
and dp_rank is not None
):
# TensorRT-LLM#16815 records this explicit first-turn placement as the
# conversation binding. On later turns, the recorded binding takes
# precedence if Dynamo supplies a different rank.
scheduling_params = SchedulingParams(
attention_dp_rank=dp_rank,
attention_dp_relax=False,
)
logging.debug(
"Using dynamo router dp_rank=%s for initial conversation-affinity "
"placement",
dp_rank,
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
elif dp_rank is not None:
scheduling_params = SchedulingParams(
attention_dp_rank=dp_rank,
Expand Down
62 changes: 61 additions & 1 deletion components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1091,11 +1091,17 @@ class TestConversationAffinity:
``SchedulingParams`` as before.
"""

def _make_handler(self, *, conversation_affinity: bool) -> HandlerBase:
def _make_handler(
self,
*,
conversation_affinity: bool,
dp_rank_source: str = "engine",
) -> HandlerBase:
config = MagicMock()
config.shutdown_event = None
config.disaggregation_mode = DisaggregationMode.AGGREGATED
config.conversation_affinity = False
config.conversation_affinity_dp_rank_source = dp_rank_source
handler = _ConcreteHandler(config)
handler.publisher = None
handler.multimodal_processor = None
Expand Down Expand Up @@ -1204,6 +1210,60 @@ async def test_affinity_on_with_session_id_suppresses_rank(self, monkeypatch):
assert conv_params is not None
assert conv_params.conversation_id == "run-42:agent-0"

@pytest.mark.asyncio
async def test_affinity_on_with_dynamo_rank_source_forwards_rank(self, monkeypatch):
"""Dynamo-owned placement forwards both the rank and conversation id."""
monkeypatch.setattr(
"dynamo.trtllm.conversation_affinity.ConversationParams",
_FakeConversationParams,
)
handler = self._make_handler(
conversation_affinity=True,
dp_rank_source="dynamo",
)
kwargs = await self._drive(
handler,
{
"token_ids": [1, 2, 3],
"stop_conditions": {"max_tokens": 10},
"sampling_options": {"temperature": 0.7},
"routing": {"dp_rank": 3},
"agent_context": {"session_id": "run-42:agent-0"},
},
)
scheduling_params = kwargs["scheduling_params"]
assert scheduling_params is not None
assert scheduling_params.attention_dp_rank == 3
assert scheduling_params.attention_dp_relax is False
conv_params = kwargs["conversation_params"]
assert conv_params is not None
assert conv_params.conversation_id == "run-42:agent-0"

@pytest.mark.asyncio
async def test_affinity_on_with_dynamo_rank_source_and_no_session_suppresses_rank(
self, monkeypatch
):
"""Without a conversation id, preserve TRT-LLM's no-id balancing path."""
monkeypatch.setattr(
"dynamo.trtllm.conversation_affinity.ConversationParams",
_FakeConversationParams,
)
handler = self._make_handler(
conversation_affinity=True,
dp_rank_source="dynamo",
)
kwargs = await self._drive(
handler,
{
"token_ids": [1, 2, 3],
"stop_conditions": {"max_tokens": 10},
"sampling_options": {"temperature": 0.7},
"routing": {"dp_rank": 3},
},
)
assert kwargs["scheduling_params"] is None
assert kwargs["conversation_params"] is None

@pytest.mark.asyncio
async def test_affinity_on_without_session_id_passes_none_conversation_params(
self, monkeypatch
Expand Down
34 changes: 34 additions & 0 deletions components/src/dynamo/trtllm/tests/test_trtllm_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,40 @@ def test_conversation_affinity_defaults_false(monkeypatch):
assert config.conversation_affinity is False


def test_conversation_affinity_dp_rank_source_cli_flag(monkeypatch):
"""The initial affinity placement owner can be selected on the CLI."""
monkeypatch.delenv(
"DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE",
raising=False,
)
config = parse_args(
[
"--model",
"fake-model",
"--conversation-affinity-dp-rank-source",
"dynamo",
]
)
assert config.conversation_affinity_dp_rank_source == "dynamo"


def test_conversation_affinity_dp_rank_source_env_var(monkeypatch):
"""DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE selects initial placement."""
monkeypatch.setenv("DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE", "dynamo")
config = parse_args(["--model", "fake-model"])
assert config.conversation_affinity_dp_rank_source == "dynamo"


def test_conversation_affinity_dp_rank_source_defaults_engine(monkeypatch):
"""TRT-LLM keeps ownership of first-turn placement by default."""
monkeypatch.delenv(
"DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE",
raising=False,
)
config = parse_args(["--model", "fake-model"])
assert config.conversation_affinity_dp_rank_source == "engine"


def test_enable_multimodal_rejects_diffusion_modality():
with pytest.raises(ValueError, match="--enable-multimodal cannot be combined"):
parse_args(
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/trtllm/tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@ def create_mock_request_handler_config(
config.kv_block_size = 32
config.shutdown_event = None
config.conversation_affinity = False
config.conversation_affinity_dp_rank_source = "engine"
config.encoder_cache_capacity_gb = encoder_cache_capacity_gb
return config
3 changes: 3 additions & 0 deletions components/src/dynamo/trtllm/workers/llm_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,9 @@ async def init_llm_worker(
max_seq_len=config.max_seq_len,
disagg_machine_id=int(endpoint.connection_id()) % 1021,
conversation_affinity=config.conversation_affinity,
conversation_affinity_dp_rank_source=(
config.conversation_affinity_dp_rank_source
),
)

media_decoder = None
Expand Down
38 changes: 38 additions & 0 deletions docs/fern/backends/trtllm/trtllm-dp-rank-routing.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,44 @@ TensorRT-LLM supports [attention data parallelism](https://lmsys.org/blog/2024-1
- **Dynamo DP Rank Routing**: The router selects the optimal DP rank based on KV cache overlap and instructs TRT-LLM to use that rank with strict routing (`attention_dp_relax=False`). Use this with `--router-mode kv` for cache-aware routing.
- **TRT-LLM Internal Routing**: TRT-LLM's scheduler assigns DP ranks internally. Use this with `--router-mode round-robin` or `random` when KV-aware routing isn't needed.

### Conversation-Aware DP Rank Routing

To keep a conversation on the same worker and attention-DP rank, enable conversation affinity in both the [Dynamo router](../../components/router/router-configuration.md#session-affinity) and the [TensorRT-LLM ADP router](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog26_DeepSeek_V4_on_NVIDIA_Blackwell_Model_Specific_and_Agentic_Workload_Optimizations_in_TensorRT-LLM.md#rank-level-routing-within-a-context-server). Dynamo passes the stable conversation ID to TensorRT-LLM through `ConversationParams`. Use `--conversation-affinity-dp-rank-source` to choose which component owns the initial attention-DP rank placement:

| Value | Initial placement | Later requests |
| --- | --- | --- |
| `engine` (default) | TensorRT-LLM selects the rank with its conversation-aware ADP router. | TensorRT-LLM reuses the stored conversation-to-rank binding. |
| `dynamo` | Dynamo forwards the KV router's selected rank as a strict `SchedulingParams.attention_dp_rank`. | TensorRT-LLM reuses the stored conversation-to-rank binding, even if a later Dynamo routing hint differs. |

Enable conversation-aware routing in the TensorRT-LLM engine configuration:

```yaml
enable_attention_dp: true
attention_dp_config:
kv_cache_routing_conversation_affinity: true
```

Then pass that configuration and configure the rank source on the Dynamo TensorRT-LLM worker:

```bash
python3 -m dynamo.trtllm \
--model-path <MODEL_PATH> \
--extra-engine-args engine.yaml \
--conversation-affinity \
--conversation-affinity-dp-rank-source engine
```

You can set the same option with the `DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE` environment variable:

```bash
export DYN_ENGINE_CONV_AFFINITY_DP_RANK_SOURCE=dynamo
```

> [!WARNING]
> The `dynamo` rank source requires a TensorRT-LLM version containing [NVIDIA/TensorRT-LLM#16815](https://github.com/NVIDIA/TensorRT-LLM/pull/16815) or equivalent support. Enable it only after Dynamo's TensorRT-LLM dependency includes that change. Without it, TensorRT-LLM may accept the explicit first-turn rank without recording it as the conversation binding, allowing later turns to move to another rank.

The rank-source option changes only who selects the first rank. Both modes require a stable conversation ID, and TensorRT-LLM owns the conversation-to-rank binding after initial placement.

### Enabling DP Rank Routing

```bash
Expand Down
Loading