Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions components/src/dynamo/kv_state_agent/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,20 @@
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Dynamo KV state-agent host")
parser.add_argument("--max-slots", type=int, default=8)
parser.add_argument(
"--slots-per-socket",
type=int,
default=1,
help=(
"Slots sharing one output PUB socket and one raw-input SUB socket. "
"Each router connection buffers one PUB's traffic for all of its slots, "
"so keep this small (for example 8): enough to stay under the ZMQ "
"socket limit with many slots, few enough that one busy router "
"connection does not overflow. Above 1, every router must support "
"shared publisher endpoints; pooled raw input needs a distinct raw "
"topic per source."
),
)
return parser.parse_args()


Expand All @@ -43,11 +57,14 @@ async def worker(runtime: DistributedRuntime) -> None:
args = parse_args()
if args.max_slots <= 0:
raise ValueError("--max-slots must be greater than zero")
if args.slots_per_socket <= 0:
raise ValueError("--slots-per-socket must be greater than zero")

namespace = os.environ.get("DYN_NAMESPACE", "dynamo")
host = KvStateAgentHost(
runtime.endpoint(f"{namespace}.kv_state_agent.control"),
args.max_slots,
args.slots_per_socket,
)
await host.start()
health_task: asyncio.Future[Any] | None = None
Expand Down
8 changes: 6 additions & 2 deletions lib/bindings/python/rust/llm/kv_state_agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,27 @@ use crate::{Endpoint, to_pyerr};
pub struct KvStateAgentHost {
endpoint: dynamo_runtime::component::Endpoint,
max_slots: usize,
slots_per_socket: usize,
inner: Arc<OnceCell<Arc<llm_rs::kv_router::publisher::KvStateAgentHost>>>,
}

#[pymethods]
impl KvStateAgentHost {
#[new]
#[pyo3(signature = (endpoint, max_slots=8))]
fn new(endpoint: Endpoint, max_slots: usize) -> Self {
#[pyo3(signature = (endpoint, max_slots=8, slots_per_socket=1))]
fn new(endpoint: Endpoint, max_slots: usize, slots_per_socket: usize) -> Self {
Self {
endpoint: endpoint.inner,
max_slots,
slots_per_socket,
inner: Arc::new(OnceCell::new()),
}
}

fn start<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let endpoint = self.endpoint.clone();
let max_slots = self.max_slots;
let slots_per_socket = self.slots_per_socket;
let inner = self.inner.clone();
crate::future_into_py(py, async move {
inner
Expand All @@ -41,6 +44,7 @@ impl KvStateAgentHost {
llm_rs::kv_router::publisher::KvStateAgentHostConfig {
endpoint,
max_slots,
slots_per_socket,
},
)
.await
Expand Down
4 changes: 3 additions & 1 deletion lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -2728,7 +2728,9 @@ class KvDcRelay:
...

class KvStateAgentHost:
def __init__(self, endpoint: Endpoint, max_slots: int = 8) -> None:
def __init__(
self, endpoint: Endpoint, max_slots: int = 8, slots_per_socket: int = 1
) -> None:
...

async def start(self) -> None:
Expand Down
Loading
Loading