diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index ef33adc168c6..9e5116f54141 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -11,7 +11,7 @@ from copy import deepcopy from dataclasses import dataclass from functools import lru_cache -from typing import Any, Protocol, cast +from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, cast from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -36,6 +36,9 @@ from .thinking import apply_default_thinking_mode_to_template_kwargs +if TYPE_CHECKING: + from vllm.config import ModelConfig + class _Renderer(Protocol): """Structural type for vLLM's chat-template renderer.""" @@ -416,16 +419,41 @@ def _validate_chat_completion_request( return validated_request +def _reasoning_parser_enabled( + reasoning_parser_class: type[ReasoningParser] | None, + chat_template_kwargs: dict[str, Any], +) -> TypeGuard[type[ReasoningParser]]: + """Whether a reasoning parser runs for this request. + + Single-sourced because three call sites must agree: _prepare_request (which + lets the parser adjust the sampling request), preprocess_chat_request (which + must know whether that parser could have rewritten the request's guidance), + and StreamingPostProcessor (which builds the parser that consumes the result). + If they drift, the sampling params get adjusted for a parser that then does not + exist and the model's control markers reach the client as visible text. + + See https://github.com/ai-dynamo/dynamo/issues/8636 for the enable_thinking + half: when the chat template runs with enable_thinking=False the reasoning + tags live in the prompt and the generated output carries none. + """ + return reasoning_parser_class is not None and ( + chat_template_kwargs.get("enable_thinking") is not False + ) + + def _prepare_request( request: dict[str, Any] | ChatCompletionRequest, *, tokenizer: TokenizerLike, tool_parser_class: type[ToolParser] | None, + reasoning_parser_class: type[ReasoningParser] | None = None, + model_config: ModelConfig | None = None, exclude_tools_when_tool_choice_none: bool = True, enable_auto_tool_choice: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, default_thinking_mode: str | None = None, validated_request: ChatCompletionRequest | None = None, + guidance_snapshots: dict[str, Any] | None = None, ) -> tuple[ChatCompletionRequest, ToolParser | None, dict[str, Any], Any, ChatParams]: """Validate request and build arguments for template rendering. @@ -497,6 +525,50 @@ def _prepare_request( ), ) + # Let the reasoning parser adjust the sampling request, mirroring the tool-parser + # path above. Parsers that split thinking from the answer on text markers need + # `skip_special_tokens=False` so those markers survive detokenisation, and they + # declare that through adjust_request(). Without this call the reasoning channel + # only works when the *client* happens to send skip_special_tokens=false. + # + # Gated identically to StreamingPostProcessor's own parser construction below: if + # the sampling params were adjusted for a parser that then does not exist, the + # markers reach the client as visible text. + # + # ReasoningParser.adjust_request is `return request` by default, so this is a no-op + # for parsers that do not need it. + if _reasoning_parser_enabled(reasoning_parser_class, chat_template_kwargs): + # Both parsers rewrite the same request, tool first and reasoning here, so a + # single before/after diff cannot say which one moved the guidance. Splitting + # it here is the only reason the caller can apply one precedence rule to a + # reasoning rewrite and a different one to a tool rewrite. + # + # Taken inside this gate, not above it: it is read only behind the same + # _reasoning_parser_enabled() test in the caller, so on the tool-only and + # no-parser paths it was pure cost. No copy either -- adjust_request() + # REPLACES request.structured_outputs with a fresh object rather than + # mutating the one this names, so the plain reference still denotes the + # pre-adjustment value afterwards. That also makes the caller's `!=` O(1) + # by identity in the common unchanged case instead of a deep structural walk. + if guidance_snapshots is not None: + # Same accessor as adjusted_structured_guidance in the caller, + # deliberately: this value is only ever compared against that one, and + # extract_structured_outputs() is a different view. + guidance_snapshots[ + "after_tool_parser" + ] = _guided_decoding_from_structured_outputs( + request_for_sampling.structured_outputs + ) + # model_config is required, not decorative: the Cohere parsers gate their + # response_format -> structural_tag rewrite on _model_config.architecture and + # return the request untouched without it, which made this whole call a no-op + # for the only shipped parser that overrides adjust_request. + request_for_sampling = reasoning_parser_class( + tokenizer, + chat_template_kwargs=chat_template_kwargs, + model_config=model_config, + ).adjust_request(request_for_sampling) + # Mistral warns that tokenize=False is unsafe for chat templates. is_mistral_tokenizer = ( tokenizer.__class__.__name__ == "MistralTokenizer" @@ -538,6 +610,8 @@ async def preprocess_chat_request( tokenizer: TokenizerLike, renderer: _Renderer, tool_parser_class: type[ToolParser] | None, + reasoning_parser_class: type[ReasoningParser] | None = None, + model_config: ModelConfig | None = None, exclude_tools_when_tool_choice_none: bool = True, enable_auto_tool_choice: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -559,6 +633,7 @@ async def preprocess_chat_request( # structured_outputs itself. Reading it afterwards would treat a # parser-generated constraint as one the caller sent and reject the request. has_explicit_output_constraint = _has_explicit_output_constraint(validated_request) + guidance_snapshots: dict[str, Any] = {} ( request_for_sampling, tool_parser, @@ -569,19 +644,31 @@ async def preprocess_chat_request( request, tokenizer=tokenizer, tool_parser_class=tool_parser_class, + reasoning_parser_class=reasoning_parser_class, + model_config=model_config, exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, enable_auto_tool_choice=enable_auto_tool_choice, default_chat_template_kwargs=default_chat_template_kwargs, default_thinking_mode=default_thinking_mode, validated_request=validated_request, + guidance_snapshots=guidance_snapshots, ) adjusted_structured_guidance = _guided_decoding_from_structured_outputs( request_for_sampling.structured_outputs ) + # Either parser's adjust_request() may rewrite the caller's constraint into the + # form the model actually speaks -- a reasoning parser converting + # response_format into a structural tag, say. Gating this recompute on + # tool_parser alone dropped a reasoning parser's rewrite on the floor: the + # request carried the adjusted constraint but the pre-adjustment copy was + # forwarded to the engine. parser_guided_decoding = ( adjusted_structured_guidance - if tool_parser is not None + if ( + tool_parser is not None + or _reasoning_parser_enabled(reasoning_parser_class, chat_template_kwargs) + ) and adjusted_structured_guidance != client_structured_guidance else None ) @@ -610,11 +697,45 @@ async def preprocess_chat_request( "tool_choice forces a tool call and cannot be combined with an " "explicit guided_* or structured_outputs constraint." ) - guided_decoding = ( - assistant_guided_decoding - if assistant_guided_decoding is not None and not is_forced_tool_choice - else tool_guided_decoding + # Which parser moved the guidance, measured rather than inferred. Both run + # inside _prepare_request against the same request -- tool first, reasoning + # second -- so the snapshot taken between them turns one ambiguous diff into + # two attributable ones: + # + # client -> after_tool_parser changed by the tool parser + # after_tool_parser -> final changed by the reasoning parser + # + # This matters because the two rewrites carry different precedence. A tool + # parser's rewrite loses to the caller's explicit constraint -- deliberate, + # pinned by test_assistant_guidance_takes_precedence_over_auto_tool_guidance + # and the response-format-precedence row of TOOL_GUIDANCE_PARITY_CASES. A + # reasoning parser's rewrite has to win, or the request carries the parser's + # structural tag while the engine is handed the pre-adjustment copy, which is + # the bug this change exists to fix. + # + # When both rewrote it there is nothing to arbitrate: the reasoning parser ran + # last, on the already-tool-adjusted request, so the final value IS the + # composition. Forwarding it is just forwarding what the request holds. + # Absent only when no tool parser ran, in which case nothing has touched the + # request yet and the caller's own constraint is the correct baseline. + # Absent whenever no reasoning parser ran, because _prepare_request only takes + # the snapshot when it is about to run one. The default is then a don't-care: + # the comparison below sits behind the same _reasoning_parser_enabled() test and + # short-circuits before reading it. + guidance_after_tool_parser = guidance_snapshots.get( + "after_tool_parser", client_structured_guidance ) + reasoning_rewrote_guidance = ( + _reasoning_parser_enabled(reasoning_parser_class, chat_template_kwargs) + and adjusted_structured_guidance != guidance_after_tool_parser + ) + guided_decoding: dict[str, Any] | None + if reasoning_rewrote_guidance and not is_forced_tool_choice: + guided_decoding = adjusted_structured_guidance + elif assistant_guided_decoding is not None and not is_forced_tool_choice: + guided_decoding = assistant_guided_decoding + else: + guided_decoding = tool_guided_decoding # build_tool_call_guided_decoding falls back to Dynamo's generic JSON schema # when a forced tool choice has no parser-provided grammar. That can happen # both without a parser and with parsers (for example Hermes) whose @@ -679,6 +800,7 @@ def __init__( tool_parser: ToolParser | None, reasoning_parser_class: type[ReasoningParser] | None, chat_template_kwargs: dict[str, Any], + model_config: ModelConfig | None = None, response_reasoning_ended: bool | None = None, stream_response: bool = True, uses_dynamo_json_tool_call_fallback: bool = False, @@ -699,14 +821,13 @@ def __init__( # `enable_thinking` is the convention adopted across the modern # reasoning-capable model families that vLLM supports; templates # that don't honor it simply leave it unset (no effect here). - thinking_disabled = chat_template_kwargs.get("enable_thinking") is False self.reasoning_parser = ( reasoning_parser_class( tokenizer, chat_template_kwargs=chat_template_kwargs, + model_config=model_config, ) - if reasoning_parser_class - and not thinking_disabled + if _reasoning_parser_enabled(reasoning_parser_class, chat_template_kwargs) and response_reasoning_ended is not True else None ) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 3208d3ec89bf..556b4b687205 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -937,7 +937,7 @@ def test_include_reasoning_false_only_marks_engine_reasoning_ended(self): from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata class FakeReasoningParser: - def __init__(self, tokenizer, *, chat_template_kwargs): + def __init__(self, tokenizer, *, chat_template_kwargs, model_config=None): self.tokenizer = tokenizer self.chat_template_kwargs = chat_template_kwargs @@ -962,7 +962,7 @@ def test_parser_receives_chat_template_kwargs(self): from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata class FakeReasoningParser: - def __init__(self, tokenizer, *, chat_template_kwargs): + def __init__(self, tokenizer, *, chat_template_kwargs, model_config=None): self.tokenizer = tokenizer self.chat_template_kwargs = chat_template_kwargs @@ -1192,6 +1192,9 @@ def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks): generation_config_fields={}, renderer=SimpleNamespace(process_for_engine_async=AsyncMock(return_value={})), process_inputs=process_inputs, + # Real InputProcessor always carries this (vllm_config.model_config); + # _generator_inner forwards it to the reasoning parser. + model_config=None, ) processor = vllm_processor_module.VllmProcessor( @@ -1231,7 +1234,7 @@ class RequestForSampling(SimpleNamespace): class FakeReasoningParser: engine_based_streaming = False - def __init__(self, tokenizer, *, chat_template_kwargs): + def __init__(self, tokenizer, *, chat_template_kwargs, model_config=None): self.tokenizer = tokenizer self.chat_template_kwargs = chat_template_kwargs self.adjusted_prompt_token_ids = None @@ -1297,6 +1300,9 @@ def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks): generation_config_fields={}, renderer=SimpleNamespace(process_for_engine_async=AsyncMock(return_value={})), process_inputs=process_inputs, + # Real InputProcessor always carries this (vllm_config.model_config); + # _generator_inner forwards it to the reasoning parser. + model_config=None, ) processor = vllm_processor_module.VllmProcessor( tokenizer=SimpleNamespace(eos_token_id=2, all_special_tokens=[]), @@ -1380,6 +1386,68 @@ async def capture_generate_and_stream( } +@pytest.mark.asyncio +async def test_generator_inner_forwards_reasoning_parser_and_model_config( + vllm_processor_module, + monkeypatch, +): + """The processor must hand its own parser AND model config to preprocessing. + + Every other reasoning-parser test calls preprocess_chat_request or + _prepare_request directly with an explicit reasoning_parser_class, so all of + them keep passing if the production call in _generator_inner stops forwarding + them. This is the boundary check that does not: it reads the arguments off the + real call site. Deleting either wiring line makes exactly this test fail. + + model_config is checked alongside because it is equally load-bearing and + equally invisible: without it the shipped Cohere parsers return the request + untouched and adjust_request() is a silent no-op. + """ + + class _SentinelReasoningParser: + """Identity is the whole point -- this is never instantiated.""" + + class _StopAfterPreprocess(Exception): + """Ends the generator at the boundary under test.""" + + sentinel_model_config = SimpleNamespace(architecture="SentinelForCausalLM") + captured: dict = {} + + async def spy_preprocess_chat_request(request, **kwargs): + captured.update(kwargs) + raise _StopAfterPreprocess + + monkeypatch.setattr( + vllm_processor_module, + "preprocess_chat_request", + spy_preprocess_chat_request, + ) + + processor = vllm_processor_module.VllmProcessor( + tokenizer=SimpleNamespace(eos_token_id=2), + input_processor=SimpleNamespace( + renderer=object(), model_config=sentinel_model_config + ), + output_processor=object(), + tool_parser_class=None, + reasoning_parser_class=_SentinelReasoningParser, + routed_engine=object(), + ) + + with pytest.raises(_StopAfterPreprocess): + await anext( + processor._generator_inner( + { + "model": "test", + "messages": [{"role": "user", "content": "Hello"}], + } + ) + ) + + assert captured.get("reasoning_parser_class") is _SentinelReasoningParser + assert captured.get("model_config") is sentinel_model_config + + def _make_processor(module, routed_engine): processor = module.VllmProcessor.__new__(module.VllmProcessor) processor.routed_engine = routed_engine @@ -1709,6 +1777,189 @@ async def test_root_thinking_suppresses_deployment_default(self, tokenizer): assert "thinking_mode" not in result.chat_template_kwargs +class _FakeStructuralTagReasoningParser: + """A reasoning parser that rewrites the caller's constraint. + + Mirrors a real text-grammar parser converting response_format into the + structural tag its model actually speaks. + """ + + def __init__(self, tokenizer, chat_template_kwargs=None, model_config=None): + del tokenizer, chat_template_kwargs + self._model_config = model_config + + def adjust_request(self, request): + request.skip_special_tokens = False + request.structured_outputs = StructuredOutputsParams( + structural_tag='{"format": "reasoning"}' + ) + return request + + +class _FakePassthroughReasoningParser(_FakeStructuralTagReasoningParser): + def adjust_request(self, request): + return request + + +class _ModelConfigGatedReasoningParser(_FakeStructuralTagReasoningParser): + """Rewrites the constraint only when it was given a model_config. + + This is the shipped Cohere parsers' actual contract: + BaseCohereCommandReasoningParser.adjust_request reads + ``self._model_config.architecture`` and returns the request untouched when the + config is absent. A fake that ignores model_config cannot tell a working call + from a no-op, which is how the missing kwarg went unnoticed. + """ + + def adjust_request(self, request): + if self._model_config is None: + return request + return super().adjust_request(request) + + +class TestReasoningParserGuidanceForwarding: + """A reasoning parser's guidance must be recomputed and forwarded. + + Regression: parser_guided_decoding was gated on `tool_parser is not None`, and + build_tool_call_guided_decoding returns early when there is no tool-call + guidance to build. A reasoning parser that rewrote the caller's constraint + therefore never surfaced: the request carried the adjusted constraint while the + engine received the pre-adjustment copy captured before _prepare_request ran. + """ + + @staticmethod + def _request(**overrides): + """A fresh request each call, so no test can perturb another's input.""" + return { + "model": MODEL, + "messages": [{"role": "user", "content": "Hello"}], + "response_format": {"type": "json_object"}, + **overrides, + } + + @staticmethod + def _renderer(): + return SimpleNamespace( + render_messages_async=AsyncMock( + return_value=(None, {"prompt_token_ids": [1]}) + ) + ) + + async def _preprocess(self, tokenizer, *, request=None, **kwargs): + kwargs.setdefault("tool_parser_class", None) + return await prepost_module.preprocess_chat_request( + request if request is not None else self._request(), + tokenizer=tokenizer, + renderer=self._renderer(), + **kwargs, + ) + + @pytest.mark.asyncio + async def test_rewrite_is_forwarded_without_a_tool_parser(self, tokenizer): + """The whole point: no tool parser anywhere in the request.""" + result = await self._preprocess( + tokenizer, + reasoning_parser_class=_FakeStructuralTagReasoningParser, + ) + + assert result.guided_decoding == {"structural_tag": '{"format": "reasoning"}'} + # The pre-adjustment copy must not win. + assert result.guided_decoding != {"json": {"type": "object"}} + + @pytest.mark.asyncio + async def test_passthrough_parser_keeps_the_caller_constraint(self, tokenizer): + """adjust_request that changes nothing must not disturb the caller.""" + result = await self._preprocess( + tokenizer, + reasoning_parser_class=_FakePassthroughReasoningParser, + ) + + assert result.guided_decoding == {"json": {"type": "object"}} + + @pytest.mark.asyncio + async def test_thinking_disabled_does_not_recompute(self, tokenizer): + """The gate is off, so no parser runs and nothing may be recomputed. + + Recomputing here would forward a constraint from a parser that the + postprocessor will not build. + """ + result = await self._preprocess( + tokenizer, + request=self._request(chat_template_kwargs={"enable_thinking": False}), + reasoning_parser_class=_FakeStructuralTagReasoningParser, + ) + + assert result.guided_decoding == {"json": {"type": "object"}} + assert result.request_for_sampling.skip_special_tokens is not False + + @pytest.mark.asyncio + async def test_rewrite_is_forwarded_with_a_tool_parser_also_active(self, tokenizer): + """Attribution is measured, so a tool parser being present is irrelevant. + + Both parsers run against the same request inside _prepare_request. The + snapshot taken between them shows the tool parser left the guidance alone + and the reasoning parser rewrote it, so the rewrite is attributable and + wins -- exactly as it does with no tool parser at all. + """ + result = await self._preprocess( + tokenizer, + request=self._request(tools=TOOL_REQUEST["tools"], tool_choice="auto"), + tool_parser_class=_FakePassthroughToolParser, + reasoning_parser_class=_FakeStructuralTagReasoningParser, + ) + + assert result.guided_decoding == {"structural_tag": '{"format": "reasoning"}'} + + @pytest.mark.asyncio + async def test_tool_parser_rewrite_still_loses_to_the_caller(self, tokenizer): + """The other half: tool-path precedence is deliberately unchanged. + + Same measurement, opposite attribution -- the tool parser moved the + guidance and the reasoning parser did not -- so the caller's explicit + constraint still wins, as pinned by + test_assistant_guidance_takes_precedence_over_auto_tool_guidance and the + response-format-precedence row of TOOL_GUIDANCE_PARITY_CASES. + """ + result = await self._preprocess( + tokenizer, + request=self._request(tools=TOOL_REQUEST["tools"], tool_choice="auto"), + tool_parser_class=_FakeAdjustRequestGrammarToolParser, + reasoning_parser_class=_FakePassthroughReasoningParser, + ) + + assert result.guided_decoding == {"json": {"type": "object"}} + + @pytest.mark.asyncio + async def test_model_config_reaches_the_parser(self, tokenizer): + """A parser that needs model_config must actually receive it. + + Without it the shipped Cohere parsers return the request untouched, which + made the whole adjust_request() call a no-op for the only parser in vLLM + that overrides it. + """ + result = await self._preprocess( + tokenizer, + reasoning_parser_class=_ModelConfigGatedReasoningParser, + model_config=SimpleNamespace(architecture="CohereForCausalLM"), + ) + + assert result.guided_decoding == {"structural_tag": '{"format": "reasoning"}'} + + @pytest.mark.asyncio + async def test_without_model_config_the_gated_parser_is_a_no_op(self, tokenizer): + """The negative control: same parser, no config, caller's constraint stands. + + This is the behaviour the PR shipped before model_config was threaded + through -- kept so the pair documents the difference. + """ + result = await self._preprocess( + tokenizer, + reasoning_parser_class=_ModelConfigGatedReasoningParser, + ) + + assert result.guided_decoding == {"json": {"type": "object"}} + + class _PassthroughStreamingToolParser(ToolParser): """A tool parser that finds no tool call and passes the raw delta through. diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index cb5ded3fe439..13ec592706ee 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -193,6 +193,7 @@ def _build_reasoning_parser_metadata( chat_template_kwargs: dict[str, Any], request_for_sampling: Any, prompt_token_ids: list[int], + model_config: Any = None, ) -> _ReasoningParserMetadata: if reasoning_parser_class is None: return _ReasoningParserMetadata(None, None, None) @@ -203,9 +204,17 @@ def _build_reasoning_parser_metadata( if getattr(request_for_sampling, "_grammar_from_tool_parser", False): return _ReasoningParserMetadata(True, True, parser_kwargs) + # Same construction as the other two sites. is_reasoning_end() does not read + # model_config, but constructing this one differently is exactly the drift that + # let the missing model_config go unnoticed at the adjust_request() site. + # + # Deliberately NOT added to parser_kwargs: that dict goes on the wire as + # dynamo_preproc["reasoning_parser_kwargs"], and ModelConfig is not + # serializable -- the worker builds its own. reasoning_parser = reasoning_parser_class( tokenizer, chat_template_kwargs=chat_template_kwargs, + model_config=model_config, ) response_reasoning_ended = reasoning_parser.is_reasoning_end(prompt_token_ids) # include_reasoning controls response projection, not whether the model may @@ -542,6 +551,8 @@ async def _generator_inner( tokenizer=self.tokenizer, renderer=self.input_processor.renderer, tool_parser_class=self.tool_parser_class, + reasoning_parser_class=self.reasoning_parser_class, + model_config=self.input_processor.model_config, exclude_tools_when_tool_choice_none=self.exclude_tools_when_tool_choice_none, enable_auto_tool_choice=self.enable_auto_tool_choice, default_chat_template_kwargs=self.default_chat_template_kwargs, @@ -644,6 +655,7 @@ async def _generator_inner( chat_template_kwargs, request_for_sampling, tokens, + self.input_processor.model_config, ) # Convert to a Python object that has fields that match our PreprocessedRequest @@ -745,6 +757,7 @@ def new_post_processor() -> StreamingPostProcessor: tool_parser=choice_tool_parser, reasoning_parser_class=self.reasoning_parser_class, chat_template_kwargs=chat_template_kwargs, + model_config=self.input_processor.model_config, response_reasoning_ended=( reasoning_metadata.response_reasoning_ended ), diff --git a/tests/frontend/test_prepost.py b/tests/frontend/test_prepost.py index c6fb1e8f75fc..af59ab8a2d0b 100644 --- a/tests/frontend/test_prepost.py +++ b/tests/frontend/test_prepost.py @@ -29,7 +29,7 @@ from vllm.sampling_params import SamplingParams from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser - from dynamo.frontend.prepost import StreamingPostProcessor + from dynamo.frontend.prepost import StreamingPostProcessor, _prepare_request else: # Fake some types so that `pre-commit` passes class CompletionOutput: @@ -2076,3 +2076,122 @@ def test_streaming_parallel_tool_calls_no_think( assert finish_reasons == [ "tool_calls" ], f"Expected finish_reason=['tool_calls']; got {finish_reasons}" + + +# --------------------------------------------------------------------------- +# Reasoning-parser adjust_request wiring +# --------------------------------------------------------------------------- +@pytest.fixture +def recording_reasoning_parser(): + """A reasoning-parser class plus the per-test list of instances it created. + + Returns ``(cls, instances)``. The list is owned by the fixture rather than + living on the class, so each test starts clean without depending on a shared + registry being cleared (see .ai/pytest-guidelines.md, "Hermetic Testing"). + """ + instances = [] + + class _RecordingReasoningParser: + """Stands in for a parser whose adjust_request mutates the request. + + Real example: a parser that splits thinking from the answer on text markers + needs skip_special_tokens=False so those markers survive detokenisation. + """ + + def __init__(self, tokenizer, chat_template_kwargs=None, model_config=None): + self.tokenizer = tokenizer + self.chat_template_kwargs = chat_template_kwargs + self.called = False + instances.append(self) + + def adjust_request(self, request): + self.called = True + request.skip_special_tokens = False + return request + + return _RecordingReasoningParser, instances + + +def _prep(tokenizer, *, reasoning_parser_class=None, chat_template_kwargs=None): + request = ChatCompletionRequest.model_construct( + messages=[{"role": "user", "content": "hi"}], + model="test", + tools=None, + tool_choice=None, + chat_template=None, + chat_template_kwargs=chat_template_kwargs, + add_generation_prompt=True, + continue_final_message=False, + documents=None, + reasoning_effort=None, + skip_special_tokens=True, + ) + return _prepare_request( + request, + tokenizer=tokenizer, + tool_parser_class=None, + reasoning_parser_class=reasoning_parser_class, + ) + + +class TestReasoningParserAdjustRequest: + """The reasoning parser's adjust_request must run during request preparation. + + Without it the reasoning channel only works when the *client* sends + skip_special_tokens=false, which ordinary OpenAI-compatible clients do not. + """ + + def test_adjust_request_is_called(self, tokenizer, recording_reasoning_parser): + parser_cls, instances = recording_reasoning_parser + req, _, _, _, _ = _prep(tokenizer, reasoning_parser_class=parser_cls) + assert len(instances) == 1 + assert instances[0].called + assert req.skip_special_tokens is False + + def test_not_called_when_thinking_disabled( + self, tokenizer, recording_reasoning_parser + ): + """Gating must match StreamingPostProcessor: no parser => no adjustment. + + If the sampling params were adjusted for a parser that then does not exist, + the model's control markers reach the client as visible text. + """ + parser_cls, instances = recording_reasoning_parser + req, _, _, _, _ = _prep( + tokenizer, + reasoning_parser_class=parser_cls, + chat_template_kwargs={"enable_thinking": False}, + ) + assert instances == [] + assert req.skip_special_tokens is True + + def test_enable_thinking_true_still_calls( + self, tokenizer, recording_reasoning_parser + ): + parser_cls, instances = recording_reasoning_parser + req, _, _, _, _ = _prep( + tokenizer, + reasoning_parser_class=parser_cls, + chat_template_kwargs={"enable_thinking": True}, + ) + assert instances[0].called + assert req.skip_special_tokens is False + + def test_no_parser_class_leaves_request_untouched(self, tokenizer): + req, _, _, _, _ = _prep(tokenizer, reasoning_parser_class=None) + assert req.skip_special_tokens is True + + def test_default_adjust_request_is_a_no_op( + self, tokenizer, recording_reasoning_parser + ): + """ReasoningParser.adjust_request returns the request unchanged by default, + so parsers that do not need the override are unaffected.""" + parser_cls, _ = recording_reasoning_parser + + class _PassThrough(parser_cls): + def adjust_request(self, request): + self.called = True + return request + + req, _, _, _, _ = _prep(tokenizer, reasoning_parser_class=_PassThrough) + assert req.skip_special_tokens is True