Skip to content
Merged
24 changes: 22 additions & 2 deletions components/src/dynamo/trtllm/request_handlers/handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from tensorrt_llm.executor.utils import RequestError
from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams
from tensorrt_llm.llmapi.llm import SamplingParams
from tensorrt_llm.llmapi.llm_args import WaitingQueuePolicy

try:
from tensorrt_llm.llmapi.llm import PreprocessedInputs
Expand Down Expand Up @@ -70,14 +71,17 @@
from dynamo.trtllm.publisher import Publisher
from dynamo.trtllm.request_handlers.base_generative_handler import BaseGenerativeHandler
from dynamo.trtllm.utils.disagg_utils import (
TRTLLM_HAS_ORIG_ELAPSED_S,
DisaggregatedParams,
DisaggregatedParamsCodec,
get_compatible_global_disagg_request_id,
)
from dynamo.trtllm.utils.request_utils import (
apply_stop_conditions_to_sampling_params,
normalize_top_k_for_trtllm,
orig_elapsed_s,
request_cache_salt,
request_trtllm_priority,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -336,6 +340,9 @@ def __init__(self, config: RequestHandlerConfig):
self._conversation_affinity_dp_rank_source: str = (
config.conversation_affinity_dp_rank_source
)
# Whether the engine's waiting queue orders by request priority
# (scheduler_config.waiting_queue_policy); resolved lazily like the gate above.
self._engine_orders_by_priority: Optional[bool] = None
self.encode_client = config.encode_client
self.multimodal_processor = config.multimodal_processor
self.first_generation = True
Expand Down Expand Up @@ -850,6 +857,11 @@ def _setup_disaggregated_params_for_mode(
disaggregated_params,
epd_metadata,
) = self._decode_disaggregated_params_from_prefill(prefill_result)
# TRT-LLM's generation admission orders this request by its arrival at
# the frontend rather than at this worker.
elapsed_s = orig_elapsed_s(request)
if elapsed_s is not None and TRTLLM_HAS_ORIG_ELAPSED_S:
disaggregated_params.orig_elapsed_s = elapsed_s
Comment thread
peilii marked this conversation as resolved.
Resolved
# For full EPD flow, make decoded params available to multimodal processor
ep_disaggregated_params = disaggregated_params

Expand Down Expand Up @@ -1264,6 +1276,13 @@ async def _generate_locally_impl(
"release newer than 1.3.0rc20)."
)
conv_affinity = self._conversation_affinity
if self._engine_orders_by_priority is None:
# Engine args without a scheduler config run the default FCFS queue.
scheduler_config = getattr(self.engine.llm.args, "scheduler_config", None)
self._engine_orders_by_priority = (
getattr(scheduler_config, "waiting_queue_policy", None)
== WaitingQueuePolicy.PRIORITY
)

# Extract dp_rank from request's routing hints for attention DP routing
routing = request.get("routing", {})
Expand Down Expand Up @@ -1300,8 +1319,9 @@ async def _generate_locally_impl(
f"Using dynamo router dp_rank={dp_rank} for TRTLLM attention DP scheduling"
)

# Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is 0.5.
priority = request.get("priority", DEFAULT_REQUEST_PRIORITY)
priority = request_trtllm_priority(
request, DEFAULT_REQUEST_PRIORITY, self._engine_orders_by_priority
)
cache_salt = request_cache_salt(request)
if cache_salt is None and request.get(HEALTH_CHECK_KEY):
# Keep the canary out of the unsalted namespace that TRT-LLM's
Expand Down
115 changes: 110 additions & 5 deletions components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from tensorrt_llm.executor.utils import RequestError
from tensorrt_llm.llmapi import DisaggregatedParams
from tensorrt_llm.llmapi.llm import SamplingParams
from tensorrt_llm.llmapi.llm_args import WaitingQueuePolicy

from dynamo.common.backend.logprobs import extract_from_completion_output
from dynamo.llm.exceptions import EngineShutdown
Expand All @@ -38,6 +39,7 @@
BYPASS_REMOTE_PREFILL_ANNOTATION,
HandlerBase,
)
from dynamo.trtllm.utils.disagg_utils import TRTLLM_HAS_ORIG_ELAPSED_S

pytestmark = [
pytest.mark.unit,
Expand Down Expand Up @@ -794,7 +796,8 @@ def test_epd_prefill_preserves_encoder_metadata(self):


class TestDisaggRequestId:
"""Tests for disagg_request_id population in _setup_disaggregated_params_for_mode."""
"""Tests for _setup_disaggregated_params_for_mode: disagg_request_id population
and the decode handoff."""

def _make_prefill_handler(self, machine_id: int = 42) -> HandlerBase:
config = MagicMock()
Expand Down Expand Up @@ -889,14 +892,73 @@ def test_decode_conditional_bypass_uses_request_disagg_params(self):
)
assert params.request_type == "context_and_generation"

@pytest.mark.parametrize(
("request_extra", "has_field", "expected"),
[
({"elapsed_since_arrival_ms": 1500.0}, True, 1.5),
({}, True, None),
({"elapsed_since_arrival_ms": 1500.0}, False, None),
],
ids=["stamped", "unstamped", "older-trtllm"],
)
def test_decode_passes_frontend_elapsed_as_orig_elapsed_s(
self, monkeypatch, request_extra, has_field, expected
):
"""A decode leg orders by its arrival at the frontend, when the router stamped it.
A TRT-LLM build without the field keeps its params untouched: they are slotted.
"""
monkeypatch.setattr(
"dynamo.trtllm.request_handlers.handler_base.TRTLLM_HAS_ORIG_ELAPSED_S",
has_field,
)
handler = self._make_decode_handler()
handler._decode_disaggregated_params_from_prefill = MagicMock(
return_value=(SimpleNamespace(request_type="generation_only"), {})
)

params, _, _ = handler._setup_disaggregated_params_for_mode(
request={"prefill_result": {"disaggregated_params": {}}, **request_extra},
ep_disaggregated_params=None,
)

assert getattr(params, "orig_elapsed_s", None) == expected

def test_decode_handoff_matches_installed_disaggregated_params(self):
"""The real params take orig_elapsed_s only on a TRT-LLM build that defines it.
Before TRT-LLM 1.4.0rc3 the slotted dataclass has no such attribute, and
setting one would raise.
"""
handler = self._make_decode_handler()
params, _, _ = handler._setup_disaggregated_params_for_mode(
request={
"elapsed_since_arrival_ms": 1500.0,
"prefill_result": {
"disaggregated_params": {
"request_type": "context_only",
"ctx_request_id": 1,
"first_gen_tokens": [7],
}
},
},
ep_disaggregated_params=None,
)

assert params.request_type == "generation_only"
expected = 1.5 if TRTLLM_HAS_ORIG_ELAPSED_S else None
assert getattr(params, "orig_elapsed_s", None) == expected


class TestGenerateLocally:
"""Verify generate_locally forwards request options to generate_async.
Health check requests (built by TrtllmHealthCheckPayload) must reach
the TRT-LLM engine at priority=1.0. Regular inference requests
(built by the Rust frontend as PreprocessedRequest, which has no
priority field) must fall back to DEFAULT_REQUEST_PRIORITY (0.5).
(built by the Rust frontend as PreprocessedRequest, which carries a
priority only under routing) reach it at routing.priority mapped onto
TRT-LLM's scale when the engine's waiting queue orders by priority, and
at DEFAULT_REQUEST_PRIORITY (0.5) otherwise.
"""

def _make_handler(self) -> HandlerBase:
Expand Down Expand Up @@ -1045,12 +1107,12 @@ async def test_health_check_gets_priority_1(self):

@pytest.mark.asyncio
async def test_regular_request_gets_default_priority(self):
"""Rust PreprocessedRequest shape (no priority key) → default 0.5."""
"""Rust PreprocessedRequest shape (no routing priority) → default 0.5."""
handler = self._make_handler()
generation_result = self._make_mock_generation_result()
handler.engine.llm.generate_async = MagicMock(return_value=generation_result)

# Mirrors the Rust PreprocessedRequest struct — no priority field.
# Mirrors the Rust PreprocessedRequest struct without routing hints.
request = {
"token_ids": [1, 2, 3],
"stop_conditions": {"max_tokens": 10},
Expand All @@ -1065,6 +1127,49 @@ async def test_regular_request_gets_default_priority(self):
_, kwargs = handler.engine.llm.generate_async.call_args
assert kwargs["priority"] == DEFAULT_REQUEST_PRIORITY

@pytest.mark.asyncio
@pytest.mark.parametrize(
("waiting_queue_policy", "expected"),
[
(WaitingQueuePolicy.PRIORITY, 0.5 + 2**-24),
(WaitingQueuePolicy.FCFS, DEFAULT_REQUEST_PRIORITY),
(None, DEFAULT_REQUEST_PRIORITY),
],
ids=["priority-queue", "fcfs-queue", "no-scheduler-config"],
)
async def test_routing_priority_follows_engine_waiting_queue(
self, waiting_queue_policy, expected
):
"""routing.priority reaches a priority-queue engine on TRT-LLM's scale.
The default FCFS queue ignores priorities and warns for each request
that carries one, so it keeps the default. Engine args without a
scheduler config run that default queue.
"""
handler = self._make_handler()
if waiting_queue_policy is None:
del handler.engine.llm.args.scheduler_config
else:
handler.engine.llm.args.scheduler_config.waiting_queue_policy = (
waiting_queue_policy
)
generation_result = self._make_mock_generation_result()
handler.engine.llm.generate_async = MagicMock(return_value=generation_result)
request = {
"token_ids": [1, 2, 3],
"stop_conditions": {"max_tokens": 10},
"sampling_options": {},
"routing": {"priority": 1},
}

context = self._make_context()
chunks = [c async for c in handler.generate_locally(request, context)]
assert len(chunks) > 0

handler.engine.llm.generate_async.assert_called_once()
_, kwargs = handler.engine.llm.generate_async.call_args
assert kwargs["priority"] == expected

@pytest.mark.asyncio
@pytest.mark.parametrize(
"error",
Expand Down
98 changes: 98 additions & 0 deletions components/src/dynamo/trtllm/tests/test_trtllm_request_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,15 @@

from types import SimpleNamespace

import numpy as np
import pytest

from dynamo.trtllm.utils.request_utils import (
apply_stop_conditions_to_sampling_params,
dynamo_priority_to_trtllm,
orig_elapsed_s,
request_cache_salt,
request_trtllm_priority,
stored_event_cache_salt,
)

Expand Down Expand Up @@ -189,3 +193,97 @@ def test_no_stop_trim_leaves_stopping_to_frontend() -> None:
# TRT-LLM maps ignore_eos to end_id=-1 and an empty executor stop-word list.
assert sampling_params.ignore_eos is True
assert sampling_params.min_tokens == 2


INT32_MIN = -(2**31)
INT32_MAX = 2**31 - 1
PRIORITY_LIMIT = 2**22
PRIORITY_STEP = 2**-24


def _as_trtllm_float(priority: int) -> np.float32:
"""TRT-LLM stores request priority as a C++ float."""
return np.float32(dynamo_priority_to_trtllm(priority))


def test_dynamo_priority_zero_maps_to_trtllm_default():
assert dynamo_priority_to_trtllm(0) == 0.5


@pytest.mark.parametrize(
"priorities",
[
range(-1000, 1001),
range(PRIORITY_LIMIT - 1000, PRIORITY_LIMIT + 1),
range(-PRIORITY_LIMIT, -PRIORITY_LIMIT + 1001),
],
)
def test_dynamo_priority_mapping_is_strict_in_float32(priorities):
mapped = [_as_trtllm_float(p) for p in priorities]
assert all(a < b for a, b in zip(mapped, mapped[1:]))
assert all(
value == dynamo_priority_to_trtllm(p) for p, value in zip(priorities, mapped)
)


@pytest.mark.parametrize(
("priority", "expected"),
[
(INT32_MAX, 0.75),
(PRIORITY_LIMIT + 1, 0.75),
(INT32_MIN, 0.25),
(-PRIORITY_LIMIT - 1, 0.25),
],
)
def test_dynamo_priority_saturates_below_health_check(priority, expected):
assert _as_trtllm_float(priority) == expected
assert _as_trtllm_float(priority) < np.float32(1.0)


@pytest.mark.parametrize(
("request_body", "engine_orders_by_priority", "expected"),
[
({"priority": 1.0, "routing": {"priority": -5}}, True, 1.0),
({"priority": 1.0}, False, 1.0),
({"routing": {"priority": 1}}, True, 0.5 + PRIORITY_STEP),
({"routing": {"priority": -1}}, True, 0.5 - PRIORITY_STEP),
({"routing": {"priority": 1}}, False, 0.5),
({"routing": {"priority": None}}, True, 0.5),
({"routing": None}, True, 0.5),
({}, True, 0.5),
],
ids=[
"top-level-wins",
"top-level-without-priority-queue",
"routing-mapped",
"routing-mapped-negative",
"fcfs-queue-keeps-default",
"no-routing-priority",
"null-routing",
"no-routing",
],
)
def test_request_trtllm_priority(request_body, engine_orders_by_priority, expected):
assert (
request_trtllm_priority(
request_body,
default=0.5,
engine_orders_by_priority=engine_orders_by_priority,
)
== expected
)


@pytest.mark.parametrize(
("request_body", "expected"),
[
({"elapsed_since_arrival_ms": 1500.0}, 1.5),
({"elapsed_since_arrival_ms": 0.0}, 0.0),
({"elapsed_since_arrival_ms": None}, None),
({}, None),
],
)
def test_orig_elapsed_s_converts_frontend_stamp(
request_body: dict, expected: float | None
) -> None:
assert orig_elapsed_s(request_body) == expected
9 changes: 9 additions & 0 deletions components/src/dynamo/trtllm/utils/disagg_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ def get_compatible_global_disagg_request_id(machine_id: int) -> int:
return _trtllm_get_global_disagg_request_id(machine_id)


# TRT-LLM 1.4.0rc3 added ``DisaggregatedParams.orig_elapsed_s``, which orders generation
# admission by a request's original arrival. The dataclass uses slots, so an older build
# rejects the attribute. The pinned build has it; the check keeps a worker installed on an
# older TRT-LLM build serving decode requests.
TRTLLM_HAS_ORIG_ELAPSED_S = "orig_elapsed_s" in {
Comment thread
peilii marked this conversation as resolved.
Resolved
field.name for field in dataclasses.fields(DisaggregatedParams)
}


class DisaggregatedParamsCodec:
"""
Codec for encoding and decoding disaggregated params for network transfer.
Expand Down
Loading
Loading