Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 79 additions & 7 deletions components/src/dynamo/frontend/sglang_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import logging
import math
import os
import re
import time
from collections.abc import AsyncGenerator
from concurrent.futures import ProcessPoolExecutor
Expand All @@ -24,7 +25,7 @@
from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine
from dynamo.llm.exceptions import InvalidArgument, Unknown
from dynamo.llm.exceptions import HttpError, InvalidArgument, Unknown

from .sglang_prepost import (
ReasoningParser,
Expand All @@ -42,6 +43,8 @@
from .thinking import runtime_default_thinking_mode
from .utils import (
PreprocessError,
as_error_envelope,
backend_invalid_argument_to_http_error,
extract_mm_urls,
handle_engine_error,
make_internal_error,
Expand All @@ -55,6 +58,42 @@
logger = logging.getLogger(__name__)


# Map SGLang/backend error strings to client-facing HTTP errors so the reason
# reaches the client instead of an opaque 500. The 400/429 status is honoured
# only where the HTTP layer can still set it -- non-streaming requests, and
# streaming with DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS (pre-commit error peek).
# Default streaming commits 200 up front, so a classified backend error there
# surfaces as 200 + SSE error event carrying the reason.
# See GitHub issue #14354 (DEP: unified semantic error propagation).
_SGL_OVERCONTEXT_RE = re.compile(
r"longer than the model's context length"
r"|exceeds\s+context"
r"|input is too long"
r"|context length",
re.IGNORECASE,
)
_SGL_QUEUE_OVERFLOW_RE = re.compile(
r"queue[^.\n]*limit reached|policy class.*queue",
re.IGNORECASE,
)


def _classify_sglang_error(message: str) -> HttpError | None:
"""Map a backend or routed-engine error message to a client-facing HttpError.

Returns None when the message is none of the recognized shapes, so the
caller keeps its existing fallback (an SSE error envelope or Unknown).
"""
err = backend_invalid_argument_to_http_error(RuntimeError(message))
if err is not None:
return err
if _SGL_OVERCONTEXT_RE.search(message):
return HttpError(400, message)
if _SGL_QUEUE_OVERFLOW_RE.search(message):
return HttpError(429, message)
return None


def _cached_tokens_from_usage(usage: dict[str, Any] | None) -> int | None:
if not isinstance(usage, dict):
return None
Expand Down Expand Up @@ -716,6 +755,13 @@ async def _generate_and_stream(
post_proc_total_ms = 0.0
created_ts = int(time.time())
stream_interval = self.stream_interval
# True once any frame has been yielded to the client. A classified
# backend error arriving before the first byte `raise`s HttpError so the
# HTTP layer can set the status where it still can (non-streaming, or
# streaming with DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS); one arriving
# mid-stream, after the 200 is committed, is sent as an SSE error event
# so the client sees the reason.
yielded_any = False

try:
dynamo_stream = await self.routed_engine.generate(
Expand Down Expand Up @@ -851,7 +897,15 @@ def flush_pending(
request_id,
message,
)
yield make_internal_error(request_id, message)
err = _classify_sglang_error(message)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For streaming requests with the default configuration, this raise still becomes an annotated backend-error frame in PythonAsyncEngine; the HTTP service commits 200 immediately when DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS is unset (its documented default). Thus an over-context or queue rejection before the first output is returned as an SSE error after HTTP 200, not the promised 400/429.

🤖 AI Fix

Enable pre-commit inspection for these request-validation errors by making the HTTP error-peek path active by default, or otherwise add a pre-stream error channel that can set the HTTP status before the SSE response is committed.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed, and thanks — this is correct, and I had it wrong in the original PR description.

In default streaming (DYN_HTTP_PRE_COMMIT_ERROR_PEEK_MS unset) the HTTP service commits 200 immediately (lib/llm/src/http/service/openai.rs ~2966: "Env var unset → skip peek, commit HTTP 200 immediately … Backend errors will surface as SSE error frames via monitor_for_disconnects"), so a pre-stream raise here surfaces as 200 + SSE error event, not a 4xx. The 400/429 status is set only where the HTTP layer can still set it: non-streaming requests (raise routed to from_http_error → StatusCode), and streaming with the pre-commit error peek enabled (check_for_backend_error peeks the first non-annotation event before commit).

So this PR's actual delivery is: the reason reaches the client on all paths (was an opaque 500 / bare SSE frame with no reason), and the 400/429 HTTP status is honoured only on non-streaming and peek-on streaming. I've corrected the code comment and the PR body / commit message to state the default-streaming behaviour explicitly, and flagged "enable the pre-commit error peek by default for these validation errors" as a follow-up — it's a Rust HTTP-service default change, which deliberately stays out of scope here (it's in DEP #14354's territory). Happy to open that follow-up separately if maintainers want it.

if err is not None:
if not yielded_any:
raise err
yield as_error_envelope(
make_internal_error(request_id, err.message)
)
break
yield as_error_envelope(make_internal_error(request_id, message))
break
engine_response = dynamo_response.data()

Expand All @@ -864,7 +918,9 @@ def flush_pending(
not isinstance(engine_response, dict)
or "token_ids" not in engine_response
):
yield handle_engine_error(engine_response, request_id, logger)
yield as_error_envelope(
handle_engine_error(engine_response, request_id, logger)
)
break

new_ids = engine_response["token_ids"]
Expand All @@ -887,6 +943,7 @@ def flush_pending(
stop_terminated=False,
engine_data=None,
)
yielded_any = True
yield envelope
if post.locally_finished:
break
Expand Down Expand Up @@ -924,16 +981,31 @@ def flush_pending(
stop_terminated=stop_terminated,
engine_data=engine_data,
)
yielded_any = True
yield envelope
if post.locally_finished:
break
except Unknown:
raise
except HttpError:
raise
except Exception as e:
logger.exception("Error generating response for request %s", request_id)
raise Unknown(
f"Error generating response for request {request_id}: {e}"
) from e
err = _classify_sglang_error(str(e))
if err is not None:
logger.warning(
"Backend rejected request %s with %d: %s",
request_id,
err.code,
err.message,
)
if not yielded_any:
raise err from e
yield as_error_envelope(make_internal_error(request_id, err.message))
else:
logger.exception("Error generating response for request %s", request_id)
raise Unknown(
f"Error generating response for request {request_id}: {e}"
) from e
finally:
if self.debug_perf and token_count > 0:
logger.info(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
SglangPreprocessWorkerResult,
SglangProcessor,
_build_dynamo_preproc,
_classify_sglang_error,
_init_worker,
_load_chat_template,
_map_finish_reason,
Expand All @@ -68,7 +69,7 @@
random_call_id,
random_uuid,
)
from dynamo.llm.exceptions import InvalidArgument
from dynamo.llm.exceptions import HttpError, InvalidArgument

# Needs sglang packages (gpu_1 container), but does not allocate GPU VRAM.
pytestmark = [
Expand Down Expand Up @@ -3315,6 +3316,42 @@ async def collect():
)


class TestClassifySglangError:
"""Classify backend/routed-engine error strings to client-facing HTTP statuses."""

def test_over_context_message_is_400(self):
err = _classify_sglang_error(
"The input (5105302 tokens) is longer than the model's "
"context length (1048576 tokens)."
)
assert isinstance(err, HttpError)
assert err.code == 400
assert "longer than the model's context length" in err.message

def test_queue_overflow_message_is_429(self):
err = _classify_sglang_error(
'router policy class "default" queue raw_isl_tokens limit reached'
" (have=5000, limit=5000)"
)
assert err is not None
assert err.code == 429
assert "queue" in err.message

def test_backend_invalid_argument_envelope_is_honoured(self):
err = _classify_sglang_error(
'BackendInvalidArgument: {"message":"min_p unsupported","code":400}'
)
assert err is not None
assert err.code == 400
assert "BackendInvalidArgument" not in err.message

@pytest.mark.parametrize(
"message", ["unknown routed_engine error", "CUDA out of memory", ""]
)
def test_unclassified_messages_return_none(self, message):
assert _classify_sglang_error(message) is None


class TestIncrementalDetokenization: # FRONTEND.6 — token-id stream → text
"""Test safe-boundary incremental detokenization."""

Expand Down Expand Up @@ -3984,16 +4021,22 @@ async def collect():

return asyncio.run(collect())

def test_routed_engine_is_error_yields_internal_error(self, tokenizer):
"""is_error() True yields a single internal_error chunk with the comment text."""
def test_routed_engine_is_error_yields_error_envelope(self, tokenizer):
"""is_error() with an unclassified message yields a tagged error frame.

An unclassified routed-engine error is wrapped in ``as_error_envelope``
so the binding reads it as an SSE error event (not a completion chunk);
an opaque 500 would otherwise drop the reason.
"""
items = self._run_stream(
tokenizer,
[FakeRoutedItem(None, is_error=True, comments=["backend disconnected"])],
)
assert len(items) == 1
err = items[0]["error"]
assert err["type"] == "internal_error"
assert "backend disconnected" in err["message"]
frame = items[0]
assert frame["_dynamo_annotated"] is True
assert frame["event"] == "error"
assert "backend disconnected" in frame["comment"][0]

def test_routed_engine_none_data_is_skipped(self, tokenizer):
"""data() is None (e.g. comment-only event) is skipped, not yielded as error."""
Expand All @@ -4008,14 +4051,17 @@ def test_routed_engine_none_data_is_skipped(self, tokenizer):
assert len(items) == 1
assert items[0]["choices"][0]["finish_reason"] == "stop"

def test_malformed_engine_response_yields_engine_error(self, tokenizer):
"""A response dict missing token_ids goes through handle_engine_error."""
def test_malformed_engine_response_yields_error_envelope(self, tokenizer):
"""A response dict missing token_ids is tagged as an error frame."""
items = self._run_stream(
tokenizer,
[{"status": "error", "message": "kv cache exhausted"}],
)
assert len(items) == 1
assert "error" in items[0]
frame = items[0]
assert frame["_dynamo_annotated"] is True
assert frame["event"] == "error"
assert "kv cache exhausted" in frame["comment"][0]

def test_completed_batches_replace_decode_context(self):
"""Completed batches replace context instead of accumulating history."""
Expand Down
Loading