Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2142ea0
fix(frontend): preflight streaming backend errors
Aphoh Jul 24, 2026
43f2ecd
fix(frontend): validate engine token budgets before streaming
Aphoh Jul 24, 2026
8628dad
test(frontend): reconcile token budget request paths
Aphoh Jul 24, 2026
f183956
fix(sglang): handle engine-less encode registration
Aphoh Jul 24, 2026
d6c301f
fix(frontend): close token budget admission gaps
Aphoh Jul 24, 2026
44741a9
fix(vllm): align frontend token budget validation
Aphoh Jul 24, 2026
569598f
Merge origin/main into warnold/fix-streaming-context-overflow-status
Aphoh Jul 24, 2026
5d7ad65
Merge remote-tracking branch 'origin/main' into warnold/fix-streaming…
Aphoh Jul 27, 2026
e649241
fix(frontend): encode backend token overflow policies
Aphoh Jul 27, 2026
5f4e942
test(frontend): compare native token budget behavior
Aphoh Jul 27, 2026
bdd13b2
test(frontend): capture TRT streaming rejection
Aphoh Jul 27, 2026
f6b3589
refactor(frontend): simplify token budget contract
Aphoh Jul 28, 2026
f4fe58a
Merge remote-tracking branch 'origin/main' into warnold/fix-streaming…
Aphoh Jul 31, 2026
8467eb6
Merge origin/main into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 5, 2026
df7cb7f
Merge branch 'main' into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 6, 2026
d9414b3
fix(trtllm): defer token budget when limit is inferred
Aphoh Aug 6, 2026
a63d228
Merge remote-tracking branch 'origin/warnold/fix-streaming-context-ov…
Aphoh Aug 6, 2026
2c9d456
Merge branch 'main' into warnold/fix-streaming-context-overflow-status
rmccorm4 Aug 7, 2026
34d6e9a
fix(frontend): initialize pooling error metric type
Aphoh Aug 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions components/src/dynamo/sglang/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@

SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake"
SPEC_DECODE_RUNTIME_KEY = "spec_decode"
# Must match `STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY` in
# `lib/llm/src/local_model/runtime_config.rs`.
STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY = "strict_request_token_limit"


def _register_model_source_path(engine: sgl.Engine, server_args: ServerArgs) -> str:
Expand Down Expand Up @@ -344,6 +347,24 @@ def _eagle_enabled_for(speculative_algorithm: Optional[str]) -> bool:
return False


def _get_strict_request_token_limit(
engine: sgl.Engine, server_args: ServerArgs
) -> Optional[int]:
"""Return SGLang's strict input-plus-output limit, if it has one.

SGLang can instead auto-truncate over-budget requests, and some modes
disable total-token validation. In either case the frontend must defer to
the engine. ``num_reserved_tokens`` covers speculative draft-token space.
"""
tokenizer_manager = engine.tokenizer_manager
if server_args.allow_auto_truncate:
return None
if not tokenizer_manager.validate_total_tokens:
return None

return max(0, tokenizer_manager.context_len - tokenizer_manager.num_reserved_tokens)


async def _get_runtime_config(
engine: sgl.Engine, server_args: ServerArgs, dynamo_args: DynamoConfig
) -> Optional[ModelRuntimeConfig]:
Expand All @@ -359,6 +380,12 @@ async def _get_runtime_config(
"""
runtime_config = ModelRuntimeConfig()
runtime_config.context_length = server_args.context_length
strict_request_token_limit = _get_strict_request_token_limit(engine, server_args)
if strict_request_token_limit is not None:
runtime_config.set_engine_specific(
STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY,
json.dumps(strict_request_token_limit),
)
# set reasoning parser and tool call parser
runtime_config.reasoning_parser = dynamo_args.dyn_reasoning_parser
runtime_config.tool_call_parser = dynamo_args.dyn_tool_call_parser
Expand Down
42 changes: 41 additions & 1 deletion components/src/dynamo/sglang/tests/test_runtime_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,36 @@ def test_eagle_enabled_for_speculative_algorithm(speculative_algorithm, expected
assert _eagle_enabled_for(speculative_algorithm) is expected


@pytest.mark.parametrize(
"allow_auto_truncate, validate_total_tokens, reserved_tokens, expected",
[
(False, True, 0, 256),
(False, True, 4, 252),
(False, True, 300, 0),
(True, True, 0, None),
(False, False, 0, None),
],
)
def test_strict_request_token_limit_matches_sglang_policy(
allow_auto_truncate, validate_total_tokens, reserved_tokens, expected
):
from dynamo.sglang.register import _get_strict_request_token_limit

engine = SimpleNamespace(
tokenizer_manager=SimpleNamespace(
context_len=256,
validate_total_tokens=validate_total_tokens,
num_reserved_tokens=reserved_tokens,
)
)
server_args = SimpleNamespace(
context_length=None,
allow_auto_truncate=allow_auto_truncate,
)

assert _get_strict_request_token_limit(engine, server_args) == expected


def test_hicache_publishes_native_offloading_capacity():
server_args = SimpleNamespace(hicache_write_policy="write_back")
assert get_hicache_native_offloading_capacity(
Expand Down Expand Up @@ -139,6 +169,7 @@ async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, capl
from dynamo.sglang import register

server_args = SimpleNamespace(
allow_auto_truncate=False,
context_length=4096,
disaggregation_mode=None,
hicache_write_policy="write_back",
Expand All @@ -154,7 +185,12 @@ async def test_hicache_publish_failure_preserves_core_capacity(monkeypatch, capl
"max_total_num_tokens": 1024,
}
engine = SimpleNamespace(
_scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info])
_scheduler_init_result=SimpleNamespace(scheduler_infos=[scheduler_info]),
tokenizer_manager=SimpleNamespace(
context_len=4096,
validate_total_tokens=True,
num_reserved_tokens=4,
),
)
capacity = SimpleNamespace(
max_num_seqs=None,
Expand Down Expand Up @@ -187,6 +223,10 @@ def fail_hicache_publish(self, key, value):

assert runtime_config.total_kv_blocks == 64
assert runtime_config.max_num_batched_tokens == 1024
assert (
runtime_config.runtime_data[register.STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY]
== "4092"
)
assert (
"Failed to attach native offloading capacity from SGLang HiCache" in caplog.text
)
14 changes: 14 additions & 0 deletions lib/llm/src/http/service/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3512,6 +3512,7 @@ mod tests {

use super::*;
use crate::discovery::ModelManagerError;
use crate::protocols::common::StopConditionsProvider;
use crate::protocols::common::extensions::NvExt;
use crate::protocols::openai::chat_completions::NvCreateChatCompletionRequest;
use crate::protocols::openai::common_ext::CommonExt;
Expand Down Expand Up @@ -3685,6 +3686,19 @@ mod tests {
}
}

#[test]
fn test_responses_max_output_tokens_reaches_chat_budget_field() {
let mut response_request = make_base_request();
response_request.inner.max_output_tokens = Some(256);

let unified_request: UnifiedRequest = response_request.try_into().unwrap();
let chat_request = unified_request.into_inner();
let stop_conditions = chat_request.extract_stop_conditions().unwrap();

assert_eq!(chat_request.inner.max_completion_tokens, Some(256));
assert_eq!(stop_conditions.max_tokens, Some(256));
}

#[test]
fn test_openai_nvext_rejects_agent_context() {
let err = serde_json::from_value::<NvExt>(serde_json::json!({
Expand Down
7 changes: 7 additions & 0 deletions lib/llm/src/local_model/runtime_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ pub use dynamo_parsers::tool_calling::StructuralTagSchemaMode;
// Reserve a topology namespace so generated taints can be rebuilt without touching caller taints.
pub const TOPOLOGY_TAINT_PREFIX: &str = "dynamo.topology/";

/// Runtime-data key for an engine-published strict input-plus-output token limit.
///
/// A backend must publish this only when requests above the limit are rejected,
/// rather than truncated or otherwise accepted. The value already accounts for
/// backend-reserved tokens.
pub const STRICT_REQUEST_TOKEN_LIMIT_RUNTIME_KEY: &str = "strict_request_token_limit";

/// Canonical worker-taint form for topology metadata.
///
/// A topology domain/value pair such as `zone=us-east-1a` becomes
Expand Down
Loading
Loading