Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9e09b79
share rust pd bootstrap normalization
sagearc Oct 7, 2026
0b99228
preserve pd routing in rust openai requests
sagearc Oct 7, 2026
fed60f9
test rust openai pd routing end to end
sagearc Oct 7, 2026
0a717d6
simplify rust pd host normalization
sagearc Oct 7, 2026
b32e8a6
simplify rust pd routing test fixture
sagearc Oct 7, 2026
c57a823
Merge branch 'main' into rust-openai-pd-routing
sagearc Oct 7, 2026
bad95c9
replace rust pd routing harness with adapter tests
sagearc Oct 7, 2026
ad917fe
use explicit imports in rust pd routing tests
sagearc Oct 7, 2026
3792097
initialize rust pd routing fields directly
sagearc Oct 7, 2026
116b73a
trim rust pd allocation test
sagearc Oct 7, 2026
5efdd90
keep choice clone budgeting in the openai adapter
sagearc Oct 8, 2026
900f4fc
keep PD routing tests focused on field preservation
sagearc Oct 8, 2026
0d6d8d8
consume PD routing metadata per prompt
sagearc Oct 8, 2026
fd8c673
explain PD routing consumption in openai handlers
sagearc Oct 8, 2026
07157f0
clarify PD routing normalization names
sagearc Oct 8, 2026
9ad462e
simplify PD routing access by prompt
sagearc Oct 8, 2026
09901f3
trim native tests from the PD routing change
sagearc Oct 8, 2026
3cc72c8
handle omitted finish reasons in the PD streaming test
sagearc Oct 8, 2026
8806eee
clarify PD routing names
sagearc Oct 8, 2026
f8edf4f
Merge branch 'main' into rust-openai-pd-routing
sagearc Oct 8, 2026
9f43434
Merge branch 'main' into rust-openai-pd-routing
sagearc Oct 9, 2026
82907a0
Merge branch 'main' into rust-openai-pd-routing
sagearc Oct 9, 2026
125ceae
merge latest base
sagearc Oct 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions rust/sglang-server/src/api_server/http/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use std::sync::Arc;
mod chat;
mod completions;
mod models;
mod pd_routing;
mod template;
mod template_builtins;
mod template_legacy;
Expand Down
26 changes: 22 additions & 4 deletions rust/sglang-server/src/api_server/http/openai/chat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use sglang_processor::{
};

use super::completions::completion_usage;
use super::pd_routing::PDRoutingFields;
use super::tools::{apply_tool_constraint, chat_delta, chat_finish_reason, parse_chat_tool_calls};
use super::{
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
Expand All @@ -53,6 +54,8 @@ struct ChatRequest {
#[serde(flatten)]
request: CreateChatCompletionRequest,
chat_template_kwargs: Option<ChatTemplateKwargs>,
#[serde(flatten)]
routing: PDRoutingFields,
}

async fn chat_completions(
Expand All @@ -62,6 +65,7 @@ async fn chat_completions(
let ChatRequest {
request,
chat_template_kwargs,
routing,
} = match body {
Ok(Json(request)) => request,
Err(rejection) => {
Expand Down Expand Up @@ -167,7 +171,16 @@ async fn chat_completions(
};

let stream = request.stream.unwrap_or(false);
let n = request.n.unwrap_or(1) as usize;
let prompt_count = 1;
let choices_per_prompt = request.n.unwrap_or(1) as usize;
let routing = match routing.normalize(prompt_count, choices_per_prompt) {
Ok(routing) => routing,
Err(error) => return openai_error(StatusCode::BAD_REQUEST, error.to_string(), false),
};
// Chat normalizes one prompt, so each bootstrap column has one entry.
let bootstrap_host = &routing.bootstrap.bootstrap_hosts[0];
let bootstrap_port = routing.bootstrap.bootstrap_ports[0];
let bootstrap_room = routing.bootstrap.bootstrap_rooms[0];
let want_logprobs = request.logprobs.unwrap_or(false);
let parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true);
let stream_tool_choice = request.tool_choice.clone();
Expand All @@ -180,17 +193,17 @@ async fn chat_completions(
.stream_options
.is_some_and(|options| options.include_usage)
|| state.server_args.stream_response_default_include_usage;
let mut submitted = Vec::with_capacity(n);
let mut submitted = Vec::with_capacity(choices_per_prompt);

// V4 prefills <think>, so the generated stream has no opening marker.
let starts_in_reasoning = matches!(
reasoning_parser.as_deref(),
Some("deepseek-v4" | "deepseek_v4" | "deepseekv4")
) && prompt.ends_with("<think>");
let mut prompt = Some(prompt);
for index in 0..n {
for index in 0..choices_per_prompt {
let rid = Rid::from_client(&format!("{response_id}-{index}"));
let choice_prompt = if index + 1 == n {
let choice_prompt = if index + 1 == choices_per_prompt {
prompt.take().expect("last chat choice owns the prompt")
} else {
prompt
Expand All @@ -210,6 +223,11 @@ async fn chat_completions(
logprob_start_len: -1,
top_logprobs_num: request.top_logprobs.unwrap_or(0) as i64,
return_text_in_logprobs: want_logprobs.then_some(true),
bootstrap_host: bootstrap_host.clone(),
bootstrap_port,
bootstrap_room,
routed_dp_rank: routing.routed_dp_rank,
disagg_prefill_dp_rank: routing.disagg_prefill_dp_rank,
..Default::default()
};
let call = match submit_generation(&state, native, stream).await {
Expand Down
36 changes: 30 additions & 6 deletions rust/sglang-server/src/api_server/http/openai/completions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ use dynamo_protocols::types::{
CreateCompletionResponse, Logprobs, Prompt, Stop,
};
use futures::StreamExt;
use serde::Deserialize;

use super::pd_routing::PDRoutingFields;
use super::{
AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream,
openai_error, submit_generation, unix_seconds_u32,
Expand All @@ -36,6 +38,14 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new().route("/v1/completions", post(completions))
}

#[derive(Deserialize)]
struct CompletionRequest {
#[serde(flatten)]
request: CreateCompletionRequest,
#[serde(flatten)]
routing: PDRoutingFields,
}

#[derive(Debug, PartialEq, Eq)]
enum PromptSpec {
Text(String),
Expand All @@ -58,9 +68,9 @@ pub(super) struct ChoiceExtensions {

async fn completions(
State(state): State<Arc<AppState>>,
body: Result<Json<CreateCompletionRequest>, JsonRejection>,
body: Result<Json<CompletionRequest>, JsonRejection>,
) -> Response {
let request = match body {
let CompletionRequest { request, routing } = match body {
Ok(Json(request)) => request,
Err(rejection) => {
return openai_error(StatusCode::BAD_REQUEST, rejection.body_text(), false);
Expand Down Expand Up @@ -127,8 +137,9 @@ async fn completions(
return openai_error(StatusCode::BAD_REQUEST, error.to_string(), false);
}

let n = request.n.unwrap_or(1) as usize;
let choice_count = match prompts.len().checked_mul(n) {
let prompt_count = prompts.len();
let choices_per_prompt = request.n.unwrap_or(1) as usize;
let choice_count = match prompt_count.checked_mul(choices_per_prompt) {
Some(count) if count <= MAX_OPENAI_CHOICES => count,
_ => {
return openai_error(
Expand All @@ -138,20 +149,28 @@ async fn completions(
);
}
};
let routing = match routing.normalize(prompt_count, choices_per_prompt) {
Ok(routing) => routing,
Err(error) => return openai_error(StatusCode::BAD_REQUEST, error.to_string(), false),
};
let response_id = format!("cmpl-{}", uuid::Uuid::new_v4().simple());
let created = unix_seconds_u32();
let mut submitted = Vec::with_capacity(choice_count);

for (prompt_index, prompt) in prompts.into_iter().enumerate() {
// Normalized routing is indexed by prompt, not by choice.
let bootstrap_host = &routing.bootstrap.bootstrap_hosts[prompt_index];
let bootstrap_port = routing.bootstrap.bootstrap_ports[prompt_index];
let bootstrap_room = routing.bootstrap.bootstrap_rooms[prompt_index];
let (text, input_ids, mut prompt_echo) = match prompt {
PromptSpec::Text(text) => {
let prompt_echo = if echo { text.clone() } else { String::new() };
(Some(text), None, prompt_echo)
}
PromptSpec::TokenIds(input_ids) => (None, Some(input_ids), String::new()),
};
for sample_index in 0..n {
let index = prompt_index * n + sample_index;
for sample_index in 0..choices_per_prompt {
let index = prompt_index * choices_per_prompt + sample_index;
let rid = Rid::from_client(&format!("{response_id}-{index}"));
if echo
&& sample_index == 0
Expand All @@ -176,6 +195,11 @@ async fn completions(
},
top_logprobs_num: request.logprobs.unwrap_or(0) as i64,
return_text_in_logprobs: request.logprobs.map(|_| true),
bootstrap_host: bootstrap_host.clone(),
bootstrap_port,
bootstrap_room,
routed_dp_rank: routing.routed_dp_rank,
disagg_prefill_dp_rank: routing.disagg_prefill_dp_rank,
..Default::default()
};
let call = match submit_generation(&state, native, stream).await {
Expand Down
129 changes: 129 additions & 0 deletions rust/sglang-server/src/api_server/http/openai/pd_routing.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
//! SGLang P/D extensions to Dynamo's standard OpenAI request types.

use serde::Deserialize;
use sglang_api_types::api::v1::{Int64OrList, OptionalInt64OrList, StringOrList};

use crate::message::request::{
BootstrapColumns, check_broadcast_budget, normalize_bootstrap_columns,
};
use crate::message::types::OneOrMany;
use crate::message::wire;
use crate::utils::error::Error;

/// Python's OpenAI PDRoutingFields, using generated API carriers. Unlike the
/// native API, OpenAI allows null list elements only for bootstrap_port.
#[derive(Deserialize)]
pub(super) struct PDRoutingFields {
bootstrap_host: Option<StringOrList>,
bootstrap_port: Option<OptionalInt64OrList>,
bootstrap_room: Option<Int64OrList>,
routed_dp_rank: Option<i64>,
disagg_prefill_dp_rank: Option<i64>,
}

/// Validated per-prompt columns, ready for admission.
pub(super) struct NormalizedPDRouting {
pub(super) bootstrap: BootstrapColumns,
pub(super) routed_dp_rank: Option<i64>,
pub(super) disagg_prefill_dp_rank: Option<i64>,
}

impl PDRoutingFields {
/// Normalize bootstrap fields per prompt and bound the handlers' host clones
/// across choices. DP hints remain scalar.
pub(super) fn normalize(
self,
prompt_count: usize,
choices_per_prompt: usize,
) -> Result<NormalizedPDRouting, Error> {
let hosts = self
.bootstrap_host
.and_then(wire::string_or_list)
.map(|hosts| match hosts {
OneOrMany::One(host) => OneOrMany::One(Some(host)),
OneOrMany::Many(hosts) => OneOrMany::Many(hosts.into_iter().map(Some).collect()),
});
let rooms = self
.bootstrap_room
.and_then(wire::int64_or_list)
.map(|rooms| match rooms {
OneOrMany::One(room) => OneOrMany::One(Some(room)),
OneOrMany::Many(rooms) => OneOrMany::Many(rooms.into_iter().map(Some).collect()),
});
let bootstrap = normalize_bootstrap_columns(
hosts,
self.bootstrap_port.and_then(wire::optional_int64_or_list),
rooms,
prompt_count,
)?;
if choices_per_prompt > 1 {
check_broadcast_budget(
bootstrap
.bootstrap_hosts
.iter()
.filter_map(Option::as_ref)
.map(String::len)
.sum(),
choices_per_prompt,
"bootstrap_host",
)?;
}
Ok(NormalizedPDRouting {
bootstrap,
routed_dp_rank: self.routed_dp_rank,
disagg_prefill_dp_rank: self.disagg_prefill_dp_rank,
})
}
}

#[cfg(test)]
mod tests {
use super::PDRoutingFields;
use serde_json::json;

#[test]
fn list_routing_preserves_per_prompt_metadata() {
// mini_lb injects scalars; the GPU fixture does not cover lists or
// distinguish nonzero DP hints in its single-rank topology.
let fields: PDRoutingFields = serde_json::from_value(json!({
"bootstrap_host": ["prefill-a", "prefill-b"],
"bootstrap_port": [8998, null],
"bootstrap_room": [9007199254740993_i64, 9007199254740995_i64],
"routed_dp_rank": 1,
"disagg_prefill_dp_rank": 3,
}))
.unwrap();
let routing = fields.normalize(2, 2).unwrap();
assert_eq!(
routing.bootstrap.bootstrap_hosts,
vec![Some("prefill-a".to_owned()), Some("prefill-b".to_owned())]
);
assert_eq!(routing.bootstrap.bootstrap_ports, vec![Some(8998), None]);
assert_eq!(
routing.bootstrap.bootstrap_rooms,
vec![Some(9007199254740993), Some(9007199254740995)]
);
assert_eq!(routing.routed_dp_rank, Some(1));
assert_eq!(routing.disagg_prefill_dp_rank, Some(3));
}

#[test]
fn null_list_elements_are_rejected_except_for_ports() {
for body in [
r#"{"bootstrap_host":[null]}"#,
r#"{"bootstrap_room":[null]}"#,
] {
assert!(serde_json::from_str::<PDRoutingFields>(body).is_err());
}
}

#[test]
fn list_hosts_cannot_bypass_choice_clone_budget() {
// 263173 bytes fit for one choice, but 255 choices exceed 64 MiB.
let fields: PDRoutingFields =
serde_json::from_value(json!({"bootstrap_host": ["x".repeat(263173)]})).unwrap();
let error = fields.normalize(1, 255).err().unwrap().to_string();
assert!(error.contains("bootstrap_host"), "{error}");
assert!(error.contains("would allocate more than"), "{error}");
}
}
58 changes: 39 additions & 19 deletions rust/sglang-server/src/message/request.rs
Original file line number Diff line number Diff line change
Expand Up @@ -263,24 +263,11 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateReques
let top_logprobs_nums = fan_out(top_logprobs_num, n, "top_logprobs_num")?;
let return_hidden = fan_out(return_hidden_states, n, "return_hidden_states")?;

// PD fields fan out like Python `_normalize_bootstrap_params`: scalars
// broadcast — except a scalar `bootstrap_room`, which becomes `room + i`
// (each item needs a distinct room; rooms are the P↔D pairing key).
// `fan_out` yields `Option<Option<T>>` for these nullable elements
// (outer: absent, inner: an explicit `null` element) — flatten, both
// mean "not set" downstream.
let bootstrap_hosts = flatten_column(fan_out(bootstrap_host, n, "bootstrap_host")?);
let bootstrap_ports = flatten_column(fan_out(bootstrap_port, n, "bootstrap_port")?);
let bootstrap_rooms = match bootstrap_room {
// `wrapping_add`, not `checked_`: rooms are drawn from `[0, 2^63)`,
// so a batch can only overflow by starting within `n` of `i64::MAX`
// — and distinct-but-wrapped still pairs P↔D, where saturating
// would collide every item onto one room.
Some(OneOrMany::One(Some(room))) => {
(0..n).map(|i| Some(room.wrapping_add(i as i64))).collect()
}
other => flatten_column(fan_out(other, n, "bootstrap_room")?),
};
let BootstrapColumns {
bootstrap_hosts,
bootstrap_ports,
bootstrap_rooms,
} = normalize_bootstrap_columns(bootstrap_host, bootstrap_port, bootstrap_room, n)?;
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
// `mm_hashes` has no batch form: honoring it only here would give the two
Expand Down Expand Up @@ -625,7 +612,7 @@ fn flatten_column<T>(column: Vec<Option<Option<T>>>) -> Vec<Option<T>> {
}

/// Reject a broadcast whose clones would exceed [`MAX_BROADCAST_CLONE_BYTES`].
pub(super) fn check_broadcast_budget(per_clone: usize, n: usize, name: &str) -> Result<(), Error> {
pub(crate) fn check_broadcast_budget(per_clone: usize, n: usize, name: &str) -> Result<(), Error> {
// `n == 1` is not a broadcast — there is one value and one prompt, so nothing
// is duplicated. Charging it here rejected ordinary single requests with a
// message about a batch they never sent.
Expand Down Expand Up @@ -666,6 +653,39 @@ fn fan_out<T: OneOrManyItem + Clone + HeapBytes>(
}
}

/// Bootstrap metadata columns with matching lengths.
pub(crate) struct BootstrapColumns {
pub(crate) bootstrap_hosts: Vec<Option<String>>,
pub(crate) bootstrap_ports: Vec<Option<i64>>,
pub(crate) bootstrap_rooms: Vec<Option<i64>>,
}

/// Normalize native and OpenAI bootstrap fields after wire decoding.
/// Missing values and null elements stay unset; lists must match the prompt count.
/// Scalars broadcast, except rooms advance once per prompt.
pub(crate) fn normalize_bootstrap_columns(
hosts: Option<OneOrMany<Option<String>>>,
ports: Option<OneOrMany<Option<i64>>>,
rooms: Option<OneOrMany<Option<i64>>>,
prompt_count: usize,
) -> Result<BootstrapColumns, Error> {
let bootstrap_hosts = flatten_column(fan_out(hosts, prompt_count, "bootstrap_host")?);
let bootstrap_ports = flatten_column(fan_out(ports, prompt_count, "bootstrap_port")?);
let bootstrap_rooms = match rooms {
// Wrapping preserves distinct pairing keys at the i64 boundary;
// saturating would make multiple requests share a room.
Some(OneOrMany::One(Some(room))) => (0..prompt_count)
.map(|i| Some(room.wrapping_add(i as i64)))
.collect(),
other => flatten_column(fan_out(other, prompt_count, "bootstrap_room")?),
};
Ok(BootstrapColumns {
bootstrap_hosts,
bootstrap_ports,
bootstrap_rooms,
})
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading
Loading