Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,11 @@ async def generate(
supported=getattr(self, "_supports_ordered_cancellation", False),
)
logprob_kwargs = self._build_logprob_kwargs(request)
routed_experts_kwargs = (
self._routed_experts_kwargs
if _nvext_extra_field_requested(request, "routed_experts")
else {}
)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔍 Expert-info smoke test needs opt-in

The expert-info smoke test sends no nvext.extra_fields but requires routed experts in three responses. The new gate makes those assertions fail until the test requests the field.

Devin Review


Was this helpful? React with 👍 or 👎 to provide feedback.

metadata_uploader = self._metadata_uploader_from_request(request)

output_options = request.get("output_options", {})
Expand Down Expand Up @@ -751,7 +756,7 @@ async def generate(
request,
thinking_budget_requested=thinking_budget_requested(request),
),
**self._routed_experts_kwargs,
**routed_experts_kwargs,
bootstrap_host=bootstrap_info["bootstrap_host"],
bootstrap_port=bootstrap_info["bootstrap_port"],
bootstrap_room=bootstrap_info["bootstrap_room"],
Expand Down Expand Up @@ -854,7 +859,7 @@ async def generate(
request,
thinking_budget_requested=thinking_budget_requested(request),
),
**self._routed_experts_kwargs,
**routed_experts_kwargs,
**mm_hashes_kwargs,
external_trace_header=trace_header,
rid=sglang_request_id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2735,6 +2735,62 @@ async def test_parallel_sampling_omits_kv_cache_hit(n):
assert reports == [expected] * n


@pytest.mark.asyncio
@pytest.mark.parametrize(
"mode", [DisaggregationMode.AGGREGATED, DisaggregationMode.DECODE]
)
@pytest.mark.parametrize(
"nvext,expected",
[
(None, False),
({"nvext": {"extra_fields": ["stop_reason"]}}, False),
({"nvext": {"extra_fields": ["routed_experts"]}}, True),
({"extra_args": {"nvext": {"extra_fields": ["routed_experts"]}}}, True),
],
)
async def test_routed_experts_requested_only_when_extra_field_opted_in(
mode, nvext, expected
):
handler = _new_decode_handler()
handler.serving_mode = mode
handler._enable_frontend_decoding = False
handler._mm_hashes_supported = False
handler._engine_supports_priority = False
handler._routed_experts_kwargs = {"return_routed_experts": True}
handler.enable_trace = False
handler._get_input_param = lambda request: {"input_ids": [1, 2]}
handler._resolve_lora = lambda request: None
chunk = {
"index": 0,
"output_ids": [42],
"meta_info": {"id": "sample-0", "finish_reason": {"type": "length"}},
}
handler.engine = SimpleNamespace(
async_generate=AsyncMock(return_value=_stream([chunk]))
)
context = SimpleNamespace(
id=lambda: "request-id",
trace_id="trace-id",
is_stopped=lambda: False,
notify_first_token=lambda: None,
)
request = {
"sampling_options": {"n": 1},
"stop_conditions": {"max_tokens": 1},
"bootstrap_info": {
"bootstrap_host": "prefill.invalid",
"bootstrap_port": 1234,
"bootstrap_room": 23,
},
**(nvext or {}),
}

[output async for output in handler.generate(request, context)]

kwargs = handler.engine.async_generate.await_args.kwargs
assert kwargs.get("return_routed_experts", False) is expected
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved


def test_prefill_dp_rank_kwargs_follows_engine_signature():
from dynamo.sglang._compat import prefill_dp_rank_kwargs

Expand Down
Loading