Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2142ea0
fix(frontend): preflight streaming backend errors
Aphoh Jul 24, 2026
43f2ecd
fix(frontend): validate engine token budgets before streaming
Aphoh Jul 24, 2026
8628dad
test(frontend): reconcile token budget request paths
Aphoh Jul 24, 2026
f183956
fix(sglang): handle engine-less encode registration
Aphoh Jul 24, 2026
d6c301f
fix(frontend): close token budget admission gaps
Aphoh Jul 24, 2026
44741a9
fix(vllm): align frontend token budget validation
Aphoh Jul 24, 2026
569598f
Merge origin/main into warnold/fix-streaming-context-overflow-status
Aphoh Jul 24, 2026
5d7ad65
Merge remote-tracking branch 'origin/main' into warnold/fix-streaming…
Aphoh Jul 27, 2026
e649241
fix(frontend): encode backend token overflow policies
Aphoh Jul 27, 2026
5f4e942
test(frontend): compare native token budget behavior
Aphoh Jul 27, 2026
bdd13b2
test(frontend): capture TRT streaming rejection
Aphoh Jul 27, 2026
f6b3589
refactor(frontend): simplify token budget contract
Aphoh Jul 28, 2026
f4fe58a
Merge remote-tracking branch 'origin/main' into warnold/fix-streaming…
Aphoh Jul 31, 2026
8467eb6
Merge origin/main into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 5, 2026
df7cb7f
Merge branch 'main' into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 6, 2026
d9414b3
fix(trtllm): defer token budget when limit is inferred
Aphoh Aug 6, 2026
a63d228
Merge remote-tracking branch 'origin/warnold/fix-streaming-context-ov…
Aphoh Aug 6, 2026
2c9d456
Merge branch 'main' into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 7, 2026
34d6e9a
fix(frontend): initialize pooling error metric type
Aphoh Aug 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions components/src/dynamo/sglang/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@

SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake"
SPEC_DECODE_RUNTIME_KEY = "spec_decode"
# Must match `STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY` in
# `lib/llm/src/local_model/runtime_config.rs`.
STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY = "strict_request_token_limit"


def _register_model_source_path(engine: sgl.Engine, server_args: ServerArgs) -> str:
Expand Down Expand Up @@ -344,6 +347,24 @@ def _eagle_enabled_for(speculative_algorithm: Optional[str]) -> bool:
return False


def _get_strict_request_token_limit(
engine: sgl.Engine, server_args: ServerArgs
) -> Optional[int]:
"""Return SGLang's strict input-plus-output limit, if it has one.

SGLang can instead auto-truncate over-budget requests, and some modes
disable total-token validation. In either case the frontend must defer to
the engine. ``num_reserved_tokens`` covers speculative draft-token space.
"""
tokenizer_manager = engine.tokenizer_manager
if server_args.allow_auto_truncate:
return None
if not tokenizer_manager.validate_total_tokens:
return None

return max(0, tokenizer_manager.context_len - tokenizer_manager.num_reserved_tokens)


async def _get_runtime_config(
engine: sgl.Engine, server_args: ServerArgs, dynamo_args: DynamoConfig
) -> Optional[ModelRuntimeConfig]:
Expand All @@ -359,6 +380,12 @@ async def _get_runtime_config(
"""
runtime_config = ModelRuntimeConfig()
runtime_config.context_length = server_args.context_length
strict_request_token_limit = _get_strict_request_token_limit(engine, server_args)
if strict_request_token_limit is not None:
runtime_config.set_engine_specific(
STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY,
json.dumps(strict_request_token_limit),
)
# set reasoning parser and tool call parser
runtime_config.reasoning_parser = dynamo_args.dyn_reasoning_parser
runtime_config.tool_call_parser = dynamo_args.dyn_tool_call_parser
Expand Down
42 changes: 41 additions & 1 deletion components/src/dynamo/sglang/tests/test_runtime_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,36 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected
assert _eagle_enabled_for(speculative_algorithm) is expected


@pytest.mark.parametrize(
"allow_auto_truncate, validate_total_tokens, reserved_tokens, expected",
[
(False, True, 0, 256),
(False, True, 4, 252),
(False, True, 300, 0),
(True, True, 0, None),
(False, False, 0, None),
],
)
def test_strict_request_token_limit_matches_sglang_policy(
allow_auto_truncate, validate_total_tokens, reserved_tokens, expected
):
from dynamo.sglang.register import _get_strict_request_token_limit

engine = SimpleNamespace(
tokenizer_manager=SimpleNamespace(
context_len=256,
validate_total_tokens=validate_total_tokens,
num_reserved_tokens=reserved_tokens,
)
)
server_args = SimpleNamespace(
context_length=None,
allow_auto_truncate=allow_auto_truncate,
)

assert _get_strict_request_token_limit(engine, server_args) == expected


def test_hicache_publishes_native_offloading_capacity():
server_args = SimpleNamespace(hicache_write_policy="write_back")
assert get_hicache_native_offloading_capacity(
Expand Down Expand Up @@ -139,6 +169,7 @@ async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, capl
from dynamo.sglang import register

server_args = SimpleNamespace(
allow_auto_truncate=False,
context_length=4096,
disaggregation_mode=None,
hicache_write_policy="write_back",
Expand All @@ -154,7 +185,12 @@ async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, capl
"max_total_num_tokens": 1024,
}
engine = SimpleNamespace(
_scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info])
_scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info]),
tokenizer_manager=SimpleNamespace(
context_len=4096,
validate_total_tokens=True,
num_reserved_tokens=4,
),
)
capacity = SimpleNamespace(
max_num_seqs=None,
Expand Down Expand Up @@ -187,6 +223,10 @@ def fail_hicache_publish(self, key, value):

assert runtime_config.total_kv_blocks == 64
assert runtime_config.max_num_batched_tokens == 1024
assert (
runtime_config.runtime_data[register.STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY]
== "4092"
)
assert (
"Failed to attach native offloading capacity from SGLang HiCache" in caplog.text
)
7 changes: 7 additions & 0 deletions lib/llm/src/local_model/runtime_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ pub use dynamo_parsers::tool_calling::StructuralTagSchemaMode;
// Reserve a topology namespace so generated taints can be rebuilt without touching caller taints.
pub const TOPOLOGY_TAINT_PREFIX: &str = "dynamo.topology/";

/// Runtime-data key for an engine-published strict input-plus-output token limit.
///
/// A backend must publish this only when requests above the limit are rejected,
/// rather than truncated or otherwise accepted. The value already accounts for
/// backend-reserved tokens.
pub const STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY: &str = "strict_request_token_limit";

/// Canonical worker-taint form for topology metadata.
///
/// A topology domain/value pair such as `zone=us-east-1a` becomes
Expand Down
161 changes: 134 additions & 27 deletions lib/llm/src/preprocessor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ use dynamo_runtime::metrics::frontend_perf::{
use std::{any::Any, collections::HashMap, pin::Pin, sync::Arc};
use tracing;

use crate::local_model::runtime_config::STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY;
#[cfg(feature = "mm-routing")]
use crate::model_card::ModelInfoType;
use crate::model_card::{ModelDeploymentCard, ModelInfo};
Expand Down Expand Up @@ -281,6 +282,10 @@ pub struct OpenAIPreprocessor {
media_loader: Option<MediaLoader>,
/// Max context length (in tokens) this model can handle, from ModelDeploymentCard
context_length: u32,
/// Engine-published strict input-plus-output limit. `None` means the
/// backend may apply another overflow policy or cannot expose an exact
/// admission budget, so total-token validation is deferred to it.
strict_request_token_limit: Option<u32>,
/// Per-image token-count engine. `None` when the feature is disabled, the
/// model isn't covered by the registry, or `preprocessor_config.json` is
/// unreadable.
Expand Down Expand Up @@ -318,22 +323,67 @@ impl OpenAIPreprocessor {
Some(context_length.saturating_sub(prompt_len as u32))
}

/// Prompt length for sizing the omitted-`max_tokens` cap. Prefers the
/// MM-expanded length; a 0 (serde-default / absent) counts as missing. With
/// images but no expanded length, defers to the backend (`None`); text-only
/// uses `token_ids_len`.
fn effective_prompt_len_for_cap(
expanded_prompt_len: Option<usize>,
has_images: bool,
/// Return the exact prompt length when the frontend can prove it.
///
/// MM routing currently expands image placeholders only. Therefore any
/// other non-empty media modality makes even an image-expanded length
/// incomplete and forces backend validation.
fn exact_prompt_len(
expanded_image_prompt_len: Option<usize>,
multi_modal_data: Option<&MultimodalDataMap>,
token_ids_len: usize,
) -> Option<usize> {
match expanded_prompt_len {
Some(n) if n > 0 => Some(n),
_ if has_images => None,
_ => Some(token_ids_len),
let has_images = multi_modal_data
.and_then(|media| media.get("image_url"))
.is_some_and(|items| !items.is_empty());
let has_other_media = multi_modal_data.is_some_and(|media| {
media
.iter()
.any(|(kind, items)| kind != "image_url" && !items.is_empty())
});

if has_other_media {
None
} else if let Some(expanded_prompt_len) =
expanded_image_prompt_len.filter(|length| *length > 0)
{
Some(expanded_prompt_len)
} else if has_images {
None
} else {
Some(token_ids_len)
}
}

/// Validate an explicit input-plus-output token budget against a strict
/// engine-published limit.
fn validate_requested_token_budget(
prompt_len: usize,
max_tokens: u32,
strict_request_token_limit: Option<u32>,
) -> Result<()> {
let Some(strict_request_token_limit) = strict_request_token_limit else {
return Ok(());
};

let requested_tokens = prompt_len.saturating_add(max_tokens as usize);
if requested_tokens > strict_request_token_limit as usize {
return Err(DynamoError::builder()
.error_type(ErrorType::InvalidArgument)
.message(format!(
"This model configuration accepts at most {} combined input and output \
tokens. However, your request has {} input tokens and asks for {} output \
tokens ({} tokens total). Please reduce the input length or requested \
output length.",
strict_request_token_limit, prompt_len, max_tokens, requested_tokens,
))
.build()
.into());
}

Ok(())
}

fn nvext_passthrough_args<R: NvExtProvider>(
request: &R,
) -> Option<serde_json::Map<String, serde_json::Value>> {
Expand Down Expand Up @@ -617,6 +667,11 @@ impl OpenAIPreprocessor {

// // Initialize runtime config from the ModelDeploymentCard
let runtime_config = mdc.runtime_config.clone();
let strict_request_token_limit = runtime_config
.get_engine_specific::<u32>(STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY)
.with_context(|| {
format!("Invalid {STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY} runtime metadata")
})?;
let kv_cache_block_size = mdc.kv_cache_block_size as usize;

// Capture MM-routing inputs before mdc is partially moved into MediaLoader.
Expand Down Expand Up @@ -796,6 +851,7 @@ impl OpenAIPreprocessor {
tool_call_parser,
media_loader,
context_length,
strict_request_token_limit,
#[cfg(feature = "mm-routing")]
image_token_counter,
#[cfg(feature = "mm-routing")]
Expand Down Expand Up @@ -930,21 +986,25 @@ impl OpenAIPreprocessor {
//
// Multimodal `token_ids` carry unexpanded image placeholders, so prefer
// the MM-expanded length when available, else defer to the backend.
let has_images = preprocessed
.multi_modal_data
.as_ref()
.and_then(|m| m.get("image_url"))
.is_some_and(|v| !v.is_empty());
let effective_prompt_len = Self::effective_prompt_len_for_cap(
let exact_prompt_len = Self::exact_prompt_len(
preprocessed
.mm_routing_info
.as_ref()
.map(|mm| mm.expanded_prompt_len),
has_images,
preprocessed.multi_modal_data.as_ref(),
preprocessed.token_ids.len(),
);
if let (Some(prompt_len), Some(max_tokens)) =
Comment thread
Aphoh marked this conversation as resolved.
Resolved
(exact_prompt_len, preprocessed.stop_conditions.max_tokens)
{
Self::validate_requested_token_budget(
prompt_len,
max_tokens,
self.strict_request_token_limit,
)?;
}
if preprocessed.stop_conditions.max_tokens.is_none()
&& let Some(prompt_len) = effective_prompt_len
&& let Some(prompt_len) = exact_prompt_len
Comment thread
Aphoh marked this conversation as resolved.
Resolved
&& let Some(max_tokens) =
Self::omitted_max_tokens_default(prompt_len, self.context_length, options)
{
Expand Down Expand Up @@ -4493,34 +4553,81 @@ mod tests {
}

#[test]
fn test_effective_prompt_len_for_cap() {
// MM-expanded length present: use it, ignoring the unexpanded token count.
fn test_exact_prompt_len() {
let images = MultimodalDataMap::from([(
"image_url".to_string(),
vec![MultimodalData::Url(
url::Url::parse("https://example.com/image.png").unwrap(),
)],
)]);
let videos = MultimodalDataMap::from([(
"video_url".to_string(),
vec![MultimodalData::Url(
url::Url::parse("https://example.com/video.mp4").unwrap(),
)],
)]);
let mixed = MultimodalDataMap::from([
(
"image_url".to_string(),
vec![MultimodalData::Url(
url::Url::parse("https://example.com/image.png").unwrap(),
)],
),
(
"audio_url".to_string(),
vec![MultimodalData::Url(
url::Url::parse("https://example.com/audio.wav").unwrap(),
)],
),
]);

// Image-expanded length present: use it instead of placeholder tokens.
assert_eq!(
OpenAIPreprocessor::effective_prompt_len_for_cap(Some(500), true, 12),
OpenAIPreprocessor::exact_prompt_len(Some(500), Some(&images), 12),
Some(500)
);
// Expanded length 0 (serde-default / absent) with images: defer to backend.
assert_eq!(
OpenAIPreprocessor::effective_prompt_len_for_cap(Some(0), true, 12),
OpenAIPreprocessor::exact_prompt_len(Some(0), Some(&images), 12),
None
);
// No routing info but images present: defer to backend.
assert_eq!(
OpenAIPreprocessor::effective_prompt_len_for_cap(None, true, 12),
OpenAIPreprocessor::exact_prompt_len(None, Some(&images), 12),
None
);
// Video/audio expansion is backend-owned, including mixed requests
// whose routing metadata expands only the image portion.
assert_eq!(
OpenAIPreprocessor::exact_prompt_len(None, Some(&videos), 12),
None
);
assert_eq!(
OpenAIPreprocessor::exact_prompt_len(Some(500), Some(&mixed), 12),
None
);
// Text-only: use the token count.
assert_eq!(
OpenAIPreprocessor::effective_prompt_len_for_cap(None, false, 12),
OpenAIPreprocessor::exact_prompt_len(None, None, 12),
Some(12)
);
// Expanded length 0 without images: fall back to the token count.
// Expanded length 0 without media: fall back to the token count.
assert_eq!(
OpenAIPreprocessor::effective_prompt_len_for_cap(Some(0), false, 12),
OpenAIPreprocessor::exact_prompt_len(Some(0), None, 12),
Some(12)
);
}

#[test]
fn test_requested_token_budget_validation() {
assert!(OpenAIPreprocessor::validate_requested_token_budget(80, 20, Some(100)).is_ok());
assert!(OpenAIPreprocessor::validate_requested_token_budget(80, 21, Some(100)).is_err());
assert!(
OpenAIPreprocessor::validate_requested_token_budget(usize::MAX, 1, Some(100)).is_err()
);
assert!(OpenAIPreprocessor::validate_requested_token_budget(usize::MAX, 1, None).is_ok());
}

#[test]
fn test_kimi_thinking_normalization_keeps_template_and_gates_in_sync() {
let template: dynamo_renderer::ChatTemplate = serde_json::from_value(serde_json::json!({
Expand Down
Loading
Loading