Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions components/src/dynamo/frontend/prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,9 +494,6 @@ def _prepare_request(
isinstance(request, dict) and request.get("thinking") is not None
),
)
# Don't let an absent top-level field clobber a nested reasoning_effort.
if request_for_sampling.reasoning_effort is None:
chat_template_kwargs.setdefault("reasoning_effort", None)

# Mistral warns that tokenize=False is unsafe for chat templates.
is_mistral_tokenizer = (
Expand Down
27 changes: 5 additions & 22 deletions components/src/dynamo/frontend/sglang_prepost.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ def _flatten_message_content(content: Any) -> Any:
return " ".join(text_parts)


def _normalize_openai_thinking_template_kwargs(
def _with_thinking_template_kwargs(
request: dict[str, Any],
default_thinking_mode: str | None = None,
) -> dict[str, Any]:
Expand All @@ -423,29 +423,12 @@ def _normalize_openai_thinking_template_kwargs(
request.get("chat_template_kwargs") or request.get("chat_template_args") or {}
)

def setdefault_reasoning(enabled: bool) -> None:
# Different SGLang model families consult different template toggles.
chat_template_kwargs.setdefault("thinking", enabled)
chat_template_kwargs.setdefault("enable_thinking", enabled)
chat_template_kwargs.setdefault(
"thinking_mode", "enabled" if enabled else "disabled"
)

thinking = request.get("thinking")
if isinstance(thinking, bool):
setdefault_reasoning(thinking)
elif isinstance(thinking, dict):
thinking_type = thinking.get("type")
if thinking_type == "enabled":
setdefault_reasoning(True)
elif thinking_type == "disabled":
setdefault_reasoning(False)

# Rust normalization already resolved every thinking control into these
# kwargs, so nothing here decides one; the deployment default below only
# applies when the request set none.
reasoning_effort = request.get("reasoning_effort")
if reasoning_effort is not None:
chat_template_kwargs["reasoning_effort"] = reasoning_effort
if reasoning_effort == "none":
setdefault_reasoning(False)

chat_template_kwargs = apply_default_thinking_mode_to_template_kwargs(
chat_template_kwargs,
Expand Down Expand Up @@ -741,7 +724,7 @@ def preprocess_chat_request(

Synchronous -- suitable for both main-process and worker-process execution.
"""
request = _normalize_openai_thinking_template_kwargs(request, default_thinking_mode)
request = _with_thinking_template_kwargs(request, default_thinking_mode)
messages = _materialize_messages(request.get("messages", []))

# Generation mode is independent of whether the client wants reasoning
Expand Down
16 changes: 16 additions & 0 deletions components/src/dynamo/frontend/tests/FRONTEND_CASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ own architecture under `components/src/dynamo/trtllm/` and is out of scope here.
- **`FRONTEND.7`** Worker subprocess boundary — preprocessing runs in a subprocess; result picklability, init, error propagation across the boundary.
- **`FRONTEND.8`** Error surface — `BackendError` / `InternalError` / engine-error handling, malformed responses, stream errors, deprecation warnings.
- **`FRONTEND.9`** Reasoning ↔ tool-call orchestration — both parsers active on the same response; distinct from `REASONING.batch.2` which is purely on output text.
- **`FRONTEND.10`** Thinking-control forwarding — the resolved `thinking` / `enable_thinking` / `thinking_mode` / `reasoning_effort` kwargs must reach the template unchanged, and identically on every backend.

## Annotation convention

Expand Down Expand Up @@ -145,3 +146,18 @@ must leave tool-call markers intact for downstream); this is the
frontend assembly view.

Examples: `TestReasoningParsing`.

## `FRONTEND.10` — Thinking-control forwarding

Rust `normalize_reasoning_template_args` resolves `thinking`,
`reasoning_effort`, and the `chat_template_args` thinking keys at the HTTP
boundary, before any backend runs. A chat processor forwards that decision;
it never makes one. A processor that derives a toggle of its own re-decides
what the frontend already settled, and the two backends then disagree for the
same request.

`_thinking_parity.py` holds the shared cases. Both backends assert against it,
so a divergence fails in one place instead of hiding until someone compares
`--dyn-chat-processor vllm` against `--dyn-chat-processor sglang` by hand.

Examples: `TestThinkingControlParity` (vllm + sglang).
62 changes: 62 additions & 0 deletions components/src/dynamo/frontend/tests/_thinking_parity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Shared behavioral cases for backend thinking-control tests.

Rust `normalize_reasoning_template_args` resolves every thinking control at the
HTTP boundary, before either processor runs, so these cases start from its
output. A processor forwards that decision; it never makes one.
"""

from dataclasses import dataclass
from typing import Any

# Model families read different keys, so a decision is written in all three.
RESOLVED_ENABLED = {
"thinking": True,
"enable_thinking": True,
"thinking_mode": "enabled",
}
RESOLVED_DISABLED = {
"thinking": False,
"enable_thinking": False,
"thinking_mode": "disabled",
}
# `adaptive` defers to the model, so it carries no toggle.
RESOLVED_ADAPTIVE = {"thinking_mode": "adaptive"}


@dataclass(frozen=True)
class ThinkingParityCase:
name: str
# Fields added to a minimal chat request, as the frontend hands them over.
request: dict[str, Any]
# Template kwargs every backend must produce for that request.
expected: dict[str, Any]


THINKING_PARITY_CASES = (
ThinkingParityCase(
"enabled", {"chat_template_args": RESOLVED_ENABLED}, RESOLVED_ENABLED
),
ThinkingParityCase(
"disabled", {"chat_template_args": RESOLVED_DISABLED}, RESOLVED_DISABLED
),
ThinkingParityCase(
"adaptive", {"chat_template_args": RESOLVED_ADAPTIVE}, RESOLVED_ADAPTIVE
),
# `none` is the regression this guards: it used to put a toggle back.
# `high` pins the documented promise that a grade still reaches graders.
*(
ThinkingParityCase(
f"adaptive-ignores-effort-{effort}",
{
"chat_template_args": {**RESOLVED_ADAPTIVE, "reasoning_effort": effort},
"reasoning_effort": effort,
},
{**RESOLVED_ADAPTIVE, "reasoning_effort": effort},
)
for effort in ("none", "high")
),
ThinkingParityCase("no-controls", {}, {}),
)
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import pytest
from _routed_engine_fakes import FakeRoutedEngine, FakeRoutedItem
from _thinking_parity import RESOLVED_DISABLED, RESOLVED_ENABLED, THINKING_PARITY_CASES
from _tool_guidance_parity import (
TOOL_GUIDANCE_PARITY_CASES,
assistant_response_format,
Expand Down Expand Up @@ -1094,17 +1095,9 @@ def __init__(self, *, model_type, stream_reasoning, force_reasoning):
self.force_reasoning = force_reasoning


@pytest.mark.parametrize(
"request_update",
[
{"thinking": False},
{"thinking": {"type": "disabled"}},
{"reasoning_effort": "none"},
],
)
def test_minimax_m3_openai_disabled_thinking_sets_thinking_mode(
request_update, monkeypatch
):
def test_minimax_m3_openai_disabled_thinking_sets_thinking_mode(monkeypatch):
"""`thinking:false`, `{"type":"disabled"}` and `reasoning_effort:none` all
arrive here as the same resolved kwargs, so one case covers them."""
monkeypatch.setattr(
sglang_prepost_module,
"ReasoningParser",
Expand All @@ -1114,8 +1107,8 @@ def test_minimax_m3_openai_disabled_thinking_sets_thinking_mode(
request = {
"model": MODEL,
"messages": [{"role": "user", "content": "Hello"}],
"chat_template_args": RESOLVED_DISABLED,
}
request.update(request_update)

class CapturingTokenizer:
chat_template = "template"
Expand Down Expand Up @@ -2417,7 +2410,7 @@ def encode(self, prompt):
request = {
"model": "deepseek-ai/DeepSeek-V4-Pro",
"messages": [{"role": "user", "content": "Hello"}],
"thinking": {"type": "enabled"},
"chat_template_args": {**RESOLVED_ENABLED, "reasoning_effort": "max"},
"reasoning_effort": "max",
}

Expand Down Expand Up @@ -2447,7 +2440,7 @@ def apply_chat_template(self, messages, **kwargs):
request = {
"model": "generic-model",
"messages": [{"role": "user", "content": "Hello"}],
"thinking": {"type": "enabled"},
"chat_template_args": RESOLVED_ENABLED,
}

result = preprocess_chat_request(
Expand All @@ -2466,16 +2459,7 @@ def apply_chat_template(self, messages, **kwargs):
assert result.request["chat_template_kwargs"]["thinking_mode"] == "enabled"
assert "chat_template_kwargs" not in request

@pytest.mark.parametrize(
"request_update",
[
{"thinking": {"type": "disabled"}},
{"reasoning_effort": "none"},
],
)
def test_reasoning_disabled_openai_inputs_set_qwen3_template_flags(
self, request_update
):
def test_reasoning_disabled_openai_inputs_set_qwen3_template_flags(self):
captured = {}

class CapturingTokenizer:
Expand All @@ -2488,8 +2472,8 @@ def apply_chat_template(self, messages, **kwargs):
request = {
"model": MODEL,
"messages": [{"role": "user", "content": "Hello"}],
"chat_template_args": RESOLVED_DISABLED,
}
request.update(request_update)

result = preprocess_chat_request(
request,
Expand Down Expand Up @@ -3787,3 +3771,26 @@ def test_qwen3_thinking_flag_changes_tokens(self, tokenizer):
tokenizer,
)
assert think.prompt_token_ids != no_think.prompt_token_ids


class TestThinkingControlParity: # FRONTEND.10
# Keep SGLang's thinking kwargs aligned with the shared backend matrix.
@pytest.mark.parametrize("case", THINKING_PARITY_CASES, ids=lambda case: case.name)
def test_shared_thinking_policy(self, case):
class CapturingTokenizer:
chat_template = "template"

def apply_chat_template(self, messages, **kwargs):
return [1, 2, 3]

result = preprocess_chat_request(
{
"model": MODEL,
"messages": [{"role": "user", "content": "Hello"}],
**case.request,
},
tokenizer=CapturingTokenizer(),
tool_call_parser_name=None,
reasoning_parser_name=None,
)
assert result.request.get("chat_template_kwargs", {}) == case.expected
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import pytest
from _routed_engine_fakes import FakeRoutedEngine as _FakeRoutedEngine
from _thinking_parity import THINKING_PARITY_CASES
from _tool_guidance_parity import (
TOOL_GUIDANCE_PARITY_CASES,
assistant_response_format,
Expand Down Expand Up @@ -2423,3 +2424,19 @@ def test_ensure_chat_template_preserves_existing_hf_template(
vllm_processor_module._ensure_chat_template(tokenizer, str(tmp_path), None)

assert tokenizer.chat_template == existing


class TestThinkingControlParity: # FRONTEND.10
# Keep vLLM's thinking kwargs aligned with the shared backend matrix.
@pytest.mark.parametrize("case", THINKING_PARITY_CASES, ids=lambda case: case.name)
def test_shared_thinking_policy(self, tokenizer, case):
_, _, kwargs, _, _ = _prepare_request(
{
"model": MODEL,
"messages": [{"role": "user", "content": "Hello"}],
**case.request,
},
tokenizer=tokenizer,
tool_parser_class=None,
)
assert kwargs == case.expected
20 changes: 10 additions & 10 deletions docs/fern/pages/reference/api/python/frontend.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ Handles:
- Reasoning content extraction via SGLang ReasoningParser
- Tool call parsing via SGLang FunctionCallParser or JsonArrayParser

[`components/src/dynamo/frontend/sglang_prepost.py#L958`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L958)
[`components/src/dynamo/frontend/sglang_prepost.py#L941`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L941)

**Public methods**

Expand All @@ -297,7 +297,7 @@ __init__(*, tokenizer, tool_call_parser: ToolCallParserType | None, reasoning_pa

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L967)
[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L950)

<h4 id="api-dynamo-frontend-sglang-prepost-sglangstreamingpostprocessor-process-output">process_output</h4>

Expand All @@ -317,7 +317,7 @@ Dict with ``token_ids`` and optional ``finish_reason``.

- `dict[str, Any] | None` — OpenAI choice dict or ``None`` if nothing to emit yet.

[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L1225)
[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L1208)
</Accordion>

<Accordion id="api-dynamo-frontend-prepost-streamingpostprocessor" title="StreamingPostProcessor (class)">
Expand All @@ -331,7 +331,7 @@ from dynamo.frontend.prepost import StreamingPostProcessor
StreamingPostProcessor(*, tokenizer: TokenizerLike, request_for_sampling: ChatCompletionRequest, sampling_params: SamplingParams, prompt_token_ids: Sequence[int], tool_parser: ToolParser | None, reasoning_parser_class: type[ReasoningParser] | None, chat_template_kwargs: dict[str, Any], response_reasoning_ended: bool | None = None, stream_response: bool = True, uses_dynamo_json_tool_call_fallback: bool = False) -> None
```

[`components/src/dynamo/frontend/prepost.py#L656`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L656)
[`components/src/dynamo/frontend/prepost.py#L653`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L653)

**Public methods**

Expand All @@ -343,7 +343,7 @@ __init__(*, tokenizer: TokenizerLike, request_for_sampling: ChatCompletionReques

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L657)
[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L654)

<h4 id="api-dynamo-frontend-prepost-streamingpostprocessor-process-output">process_output</h4>

Expand All @@ -353,7 +353,7 @@ process_output(output: Any) -> dict[str, Any] | None

No summary available.

[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L1102)
[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L1099)
</Accordion>

<Accordion id="api-dynamo-frontend-vllm-processor-vllmprocessor" title="VllmProcessor (class)">
Expand Down Expand Up @@ -434,7 +434,7 @@ from dynamo.frontend.sglang_prepost import build_response_format_guided_decoding
build_response_format_guided_decoding(request: dict[str, Any]) -> dict[str, Any] | None
```

[`components/src/dynamo/frontend/sglang_prepost.py#L640`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L640)
[`components/src/dynamo/frontend/sglang_prepost.py#L623`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L623)
</Accordion>

<Accordion id="api-dynamo-frontend-prepost-build-tool-call-guided-decoding" title="build_tool_call_guided_decoding (function)">
Expand Down Expand Up @@ -462,7 +462,7 @@ from dynamo.frontend.sglang_prepost import build_tool_call_guided_decoding
build_tool_call_guided_decoding(request: dict[str, Any], *, tool_call_parser_name: str | None, sglang_tools: list[SglangTool] | None) -> dict[str, Any] | None
```

[`components/src/dynamo/frontend/sglang_prepost.py#L563`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L563)
[`components/src/dynamo/frontend/sglang_prepost.py#L546`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L546)
</Accordion>

<Accordion id="api-dynamo-frontend-sglang-prepost-convert-tools" title="convert_tools (function)">
Expand Down Expand Up @@ -700,7 +700,7 @@ from dynamo.frontend.prepost import preprocess_chat_request
preprocess_chat_request(request: dict[str, Any] | ChatCompletionRequest, *, tokenizer: TokenizerLike, renderer: _Renderer, tool_parser_class: type[ToolParser] | None, exclude_tools_when_tool_choice_none: bool = True, enable_auto_tool_choice: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, default_thinking_mode: str | None = None, structural_tag_mode: str = 'off', structural_tag_scope: str = 'auto', structural_tag_schema: str = 'auto') -> PreprocessResult
```

[`components/src/dynamo/frontend/prepost.py#L536`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L536)
[`components/src/dynamo/frontend/prepost.py#L533`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/prepost.py#L533)
</Accordion>

<Accordion id="api-dynamo-frontend-sglang-prepost-preprocess-chat-request" title="preprocess_chat_request (function)">
Expand All @@ -721,7 +721,7 @@ request-level thinking controls.

Synchronous -- suitable for both main-process and worker-process execution.

[`components/src/dynamo/frontend/sglang_prepost.py#L725`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L725)
[`components/src/dynamo/frontend/sglang_prepost.py#L708`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/sglang_prepost.py#L708)
</Accordion>

<Accordion id="api-dynamo-frontend-utils-random-call-id" title="random_call_id (function)">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,12 @@ Models that support adaptive thinking can still select it per request:
}
```

Dynamo normalizes this request to `thinking_mode=adaptive`. Because adaptive is
an explicit request control, it takes precedence over an `enabled` or
`disabled` deployment default.
Dynamo normalizes this request to `thinking_mode=adaptive` and removes any
`thinking` or `enable_thinking` key, so the decision stays with the model. A
`reasoning_effort` value still reaches models that grade on it, but under
`adaptive` it does not turn reasoning on or off. Because adaptive is an
explicit request control, it takes precedence over an `enabled` or `disabled`
deployment default.

> [!NOTE]
> This option controls chat-template rendering. It does not force the inference
Expand Down
Loading
Loading