Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ tonic-build = { version = "0.13.1" }
kube = { version = "2.0.1", default-features = false, features = ["client", "rustls-tls", "aws-lc-rs", "runtime"] }
k8s-openapi = { version = "0.26.0", features = ["v1_32"] }
rcgen = { version = "0.13" }
quinn = { version = "0.11.9", default-features = false, features = ["runtime-tokio", "rustls-ring"] }
sha2 = { version = "0.10" }
tokio-rustls = { version = "0.26" }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
rustls-pemfile = { version = "2" }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import argparse
import asyncio
import base64
import os
import time
from collections.abc import AsyncGenerator
from typing import Any
Expand Down Expand Up @@ -54,6 +55,11 @@ async def from_args(
parser.add_argument("--endpoint-types", default="images")
parser.add_argument("--discovery-backend", default="etcd")
parser.add_argument("--request-plane", default="tcp")
parser.add_argument(
"--response-plane",
choices=["tcp", "quic"],
default=os.environ.get("DYN_RESPONSE_PLANE", "tcp"),
)
parser.add_argument("--event-plane", default=None)
args = parser.parse_args(argv)

Expand All @@ -70,6 +76,7 @@ async def from_args(
endpoint_types=args.endpoint_types,
discovery_backend=args.discovery_backend,
request_plane=args.request_plane,
response_plane=args.response_plane,
event_plane=args.event_plane,
# Diffusion has no KV cache to route on.
enable_kv_routing=False,
Expand Down
7 changes: 7 additions & 0 deletions components/src/dynamo/common/backend/sample_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import asyncio
import itertools
import logging
import os
import queue
import threading
import uuid
Expand Down Expand Up @@ -140,6 +141,11 @@ async def from_args(
parser.add_argument("--endpoint-types", default="chat,completions")
parser.add_argument("--discovery-backend", default="etcd")
parser.add_argument("--request-plane", default="tcp")
parser.add_argument(
"--response-plane",
choices=["tcp", "quic"],
default=os.environ.get("DYN_RESPONSE_PLANE", "tcp"),
)
parser.add_argument("--event-plane", default=None)
parser.add_argument(
"--disaggregation-mode",
Expand Down Expand Up @@ -176,6 +182,7 @@ async def from_args(
endpoint_types=args.endpoint_types,
discovery_backend=args.discovery_backend,
request_plane=args.request_plane,
response_plane=args.response_plane,
event_plane=args.event_plane,
disaggregation_mode=mode,
route_to_encoder=args.route_to_encoder,
Expand Down
7 changes: 7 additions & 0 deletions components/src/dynamo/common/backend/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import asyncio
import logging
import os
import signal
import warnings
from dataclasses import dataclass, field
Expand Down Expand Up @@ -144,6 +145,7 @@ class WorkerConfig:
# KV event/recovery ownership endpoint. None uses this worker's serving endpoint.
kv_state_endpoint: Optional[str] = None
default_thinking_mode: Optional[str] = None
response_plane: str = "tcp"

@classmethod
def from_runtime_config(
Expand All @@ -167,6 +169,7 @@ def from_runtime_config(
),
"discovery_backend": runtime_cfg.discovery_backend,
"request_plane": runtime_cfg.request_plane,
"response_plane": getattr(runtime_cfg, "response_plane", "tcp"),
"event_plane": runtime_cfg.event_plane,
"use_kv_events": getattr(runtime_cfg, "use_kv_events", False),
"custom_jinja_template": getattr(
Expand Down Expand Up @@ -242,6 +245,10 @@ async def run(self) -> None:
stacklevel=2,
)

if self.config.response_plane not in {"tcp", "quic"}:
raise ValueError("response_plane must be 'tcp' or 'quic'")
os.environ["DYN_RESPONSE_PLANE"] = self.config.response_plane

runtime_cfg = _backend.RuntimeConfig(
discovery_backend=self.config.discovery_backend,
request_plane=self.config.request_plane,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class DynamoRuntimeConfig(ConfigBase):
kv_state_endpoint: Optional[str] = None
discovery_backend: str
request_plane: str
response_plane: str = "tcp"
event_plane: Optional[str] = None
fpm_trace: bool = False
connector: list[str]
Expand Down Expand Up @@ -105,6 +106,8 @@ def validate(self) -> None:
f"--engine-request-limit must be a positive integer, got {self.engine_request_limit}"
)

os.environ["DYN_RESPONSE_PLANE"] = self.response_plane

# Propagate TCP TLS CLI flags to env vars so the Rust runtime picks them up.
if self.tcp_tls_cert_path:
os.environ["DYN_TCP_TLS_CERT_PATH"] = self.tcp_tls_cert_path
Expand Down Expand Up @@ -213,6 +216,14 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
help="Determines how requests are distributed from routers to workers. 'tcp' is fastest.",
choices=["tcp", "nats"],
)
add_argument(
g,
flag_name="--response-plane",
env_var="DYN_RESPONSE_PLANE",
default="tcp",
help="Select the response transport. Frontend and workers must match.",
choices=["tcp", "quic"],
)
add_argument(
g,
flag_name="--event-plane",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,30 @@ def test_frontend_reasoning_field_name_rejects_invalid_choice() -> None:
parser.parse_args(["--reasoning-field-name", "invalid"])


def test_frontend_response_plane_defaults_to_tcp_and_accepts_quic(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("DYN_RESPONSE_PLANE", raising=False)
parser = argparse.ArgumentParser()
FrontendArgGroup().add_arguments(parser)

default_config = FrontendConfig.from_cli_args(parser.parse_args([]))
quic_config = FrontendConfig.from_cli_args(
parser.parse_args(["--response-plane", "quic"])
)
monkeypatch.setenv("DYN_RESPONSE_PLANE", "quic")
env_parser = argparse.ArgumentParser()
FrontendArgGroup().add_arguments(env_parser)
env_config = FrontendConfig.from_cli_args(env_parser.parse_args([]))

assert default_config.response_plane == "tcp"
assert quic_config.response_plane == "quic"
assert env_config.response_plane == "quic"

with pytest.raises(SystemExit):
parser.parse_args(["--response-plane", "invalid"])


def test_conditional_disagg_config_cli_lowers_to_router_kwargs() -> None:
parser = argparse.ArgumentParser()
FrontendArgGroup().add_arguments(parser)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,26 @@ def test_kv_state_endpoint_supports_cli_and_env(monkeypatch):
assert "DYN_KV_STATE_ENDPOINT" in help_text


def test_response_plane_defaults_to_tcp_and_accepts_quic(monkeypatch):
monkeypatch.delenv("DYN_RESPONSE_PLANE", raising=False)

default_config, help_text = _parse_runtime_args([])
quic_config, _ = _parse_runtime_args(["--response-plane", "quic"])
monkeypatch.setenv("DYN_RESPONSE_PLANE", "quic")
env_config, _ = _parse_runtime_args([])

assert default_config.response_plane == "tcp"
assert quic_config.response_plane == "quic"
assert env_config.response_plane == "quic"
assert os.environ["DYN_RESPONSE_PLANE"] == "quic"
assert "--response-plane" in help_text


def test_response_plane_rejects_invalid_value():
with pytest.raises(SystemExit):
_parse_runtime_args(["--response-plane", "invalid"])


def test_fpm_trace_env_enables_and_is_canonicalized(monkeypatch):
monkeypatch.setenv("DYN_FPM_TRACE", "on")

Expand Down
7 changes: 7 additions & 0 deletions components/src/dynamo/common/utils/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def create_runtime(
request_plane: str,
event_plane: Optional[str] = None,
use_kv_events: Optional[bool] = None,
response_plane: str = "tcp",
) -> Tuple[DistributedRuntime, asyncio.AbstractEventLoop]:
"""Create a DistributedRuntime.

Expand All @@ -59,6 +60,8 @@ def create_runtime(
use_kv_events: Deprecated. NATS enablement is now determined automatically
from the event-plane configuration. This parameter is accepted for
backwards compatibility but will be removed in a future release.
response_plane: Response transport (tcp or quic). Frontend and workers
must use the same value.

Returns:
Tuple of (runtime, event_loop).
Expand All @@ -72,13 +75,17 @@ def create_runtime(
stacklevel=2,
)

if response_plane not in {"tcp", "quic"}:
raise ValueError("response_plane must be 'tcp' or 'quic'")

loop = asyncio.get_running_loop()

runtime = DistributedRuntime(
loop,
discovery_backend,
request_plane,
event_plane=event_plane,
response_plane=response_plane,
)

return runtime, loop
Expand Down
9 changes: 9 additions & 0 deletions components/src/dynamo/frontend/frontend_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase):

discovery_backend: str
request_plane: str
response_plane: str = "tcp"
event_plane: Optional[str] = None
chat_processor: str
enable_anthropic_api: bool
Expand Down Expand Up @@ -516,6 +517,14 @@ def add_arguments(self, parser) -> None:
),
choices=["nats", "tcp"],
)
add_argument(
g,
flag_name="--response-plane",
env_var="DYN_RESPONSE_PLANE",
default="tcp",
help="Select the response transport. Frontend and workers must match.",
choices=["tcp", "quic"],
)
add_argument(
g,
flag_name="--event-plane",
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/frontend/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,7 @@ async def async_main():
config.discovery_backend,
config.request_plane,
event_plane=config.event_plane,
response_plane=config.response_plane,
)

def signal_handler():
Expand Down
7 changes: 7 additions & 0 deletions components/src/dynamo/mocker/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
default=os.environ.get("DYN_REQUEST_PLANE", "tcp"),
help="Determines how requests are distributed from routers to workers. 'tcp' is fastest [nats|tcp]",
)
parser.add_argument(
"--response-plane",
type=str,
choices=["tcp", "quic"],
default=os.environ.get("DYN_RESPONSE_PLANE", "tcp"),
help="Select the response transport. Frontend and workers must match.",
)
parser.add_argument(
"--event-plane",
type=str,
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/mocker/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ async def launch_workers(args: argparse.Namespace, base_engine_args):
args.discovery_backend,
args.request_plane,
args.event_plane,
response_plane=args.response_plane,
)
runtimes.append(runtime)

Expand Down
12 changes: 12 additions & 0 deletions components/src/dynamo/mocker/tests/unit/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,3 +688,15 @@ def test_mock_engine_args_from_json_ignores_legacy_has_perf_model_field():
assert engine_args.max_num_seqs is None
assert engine_args.max_num_batched_tokens is None
assert engine_args.worker_type == "decode"


def test_response_plane_defaults_to_tcp_and_accepts_quic(monkeypatch):
monkeypatch.delenv("DYN_RESPONSE_PLANE", raising=False)

assert parse_args([]).response_plane == "tcp"
assert parse_args(["--response-plane", "quic"]).response_plane == "quic"
monkeypatch.setenv("DYN_RESPONSE_PLANE", "quic")
assert parse_args([]).response_plane == "quic"

with pytest.raises(SystemExit):
parse_args(["--response-plane", "invalid"])
1 change: 1 addition & 0 deletions components/src/dynamo/sglang/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ async def worker(argv: list[str] | None = None):
discovery_backend=dynamo_args.discovery_backend,
request_plane=dynamo_args.request_plane,
event_plane=dynamo_args.event_plane,
response_plane=dynamo_args.response_plane,
)

# Keep the flock alive for process lifetime. Linux releases it on exit.
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/trtllm/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ async def worker(argv: list[str] | None = None):
discovery_backend=config.discovery_backend,
request_plane=config.request_plane,
event_plane=config.event_plane,
response_plane=config.response_plane,
)
else:
_validate_supported_snapshot_config(config)
Expand Down
3 changes: 3 additions & 0 deletions components/src/dynamo/trtllm/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,15 @@ def _create_runtime(
discovery_backend: str,
request_plane: str,
event_plane: str | None,
response_plane: str = "tcp",
) -> tuple[Any, Any]:
from dynamo.common.utils.runtime import create_runtime as _create_runtime

return _create_runtime(
discovery_backend=discovery_backend,
request_plane=request_plane,
event_plane=event_plane,
response_plane=response_plane,
)


Expand Down Expand Up @@ -153,6 +155,7 @@ async def snapshot_before_endpoint(self, engine: Any, config: Any) -> None:
discovery_backend=config.discovery_backend,
request_plane=config.request_plane,
event_plane=config.event_plane,
response_plane=config.response_plane,
)
from dynamo.common.snapshot.lifecycle import elect_and_wake

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def _runtime_config(**overrides):
"namespace": "checkpoint-ns",
"discovery_backend": "kubernetes",
"request_plane": "nats",
"response_plane": "tcp",
"event_plane": None,
}
values.update(overrides)
Expand Down Expand Up @@ -134,10 +135,13 @@ async def wait_for_restore(self):
assert await self.pause_controller.pause(self.engine) is True
return True

def fake_create_runtime(discovery_backend, request_plane, event_plane):
def fake_create_runtime(
discovery_backend, request_plane, event_plane, response_plane="tcp"
):
assert discovery_backend == "kubernetes"
assert request_plane == "nats"
assert event_plane is None
assert response_plane == "tcp"
assert "resume" not in lifecycle_calls
return created_runtime, object()

Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/vllm/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ async def worker(argv: list[str] | None = None) -> None:
discovery_backend=config.discovery_backend,
request_plane=config.request_plane,
event_plane=config.event_plane,
response_plane=config.response_plane,
)

if snapshot_controller is not None:
Expand Down
1 change: 1 addition & 0 deletions components/src/dynamo/vllm/omni/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ async def worker():
discovery_backend=config.discovery_backend,
request_plane=config.request_plane,
event_plane=config.event_plane,
response_plane=config.response_plane,
)

install_signal_handlers(loop, runtime, shutdown_endpoints, shutdown_event)
Expand Down
Loading
Loading