From ad5026e9884b98b47d8a417026a418578397a3cb Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 11:33:10 -0800 Subject: [PATCH 1/3] fix(frontend): pass named passthrough fields to Python chat processors The frontend validates vLLM's named passthrough fields (stop_token_ids, allowed_token_ids, logprob_token_ids, ...) but keeps them in a map that is never serialized, so a Python chat processor (--dyn-chat-processor vllm or sglang, or any chat_engine_factory) never received them. The in-process adapter now adds them to the request dict at the top level, as the client sent them. The vLLM processor forwards them to the worker in extra_args.sampling_options, as the Rust preprocessor does, and drops the top-k logprob count when logprob_token_ids sets the width, as the worker already does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../tests/test_vllm_processor_unit.py | 89 +++++++++++++++++++ .../src/dynamo/frontend/vllm_processor.py | 29 ++++++ .../frontend/protocol-field-handling.md | 4 + .../components/frontend-configuration.mdx | 4 +- lib/bindings/python/rust/engine.rs | 10 ++- .../src/protocols/openai/chat_completions.rs | 9 ++ lib/llm/src/protocols/openai/validate.rs | 29 ++++++ 7 files changed, 172 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index a36fe75e5e2d..60d3d5ade4e5 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -1827,6 +1827,95 @@ async def capture_generate_and_stream( assert captured["sampling_options"]["min_p"] == expected_min_p +@pytest.mark.asyncio +async def test_generator_forwards_sampling_passthrough_fields( + vllm_processor_module, + monkeypatch, +): + """Passthrough fields reach the worker as the Rust preprocessor sends them.""" + + class RequestForSampling(SimpleNamespace): + model_fields = frozenset({"logprob_token_ids"}) + + monkeypatch.setattr( + vllm_processor_module, + "preprocess_chat_request", + AsyncMock( + return_value=SimpleNamespace( + request_for_sampling=RequestForSampling( + max_completion_tokens=None, + max_tokens=1, + logprobs=True, + top_logprobs=0, + cache_salt=None, + mm_processor_kwargs=None, + top_k=None, + min_p=None, + logprob_token_ids=[5, 6], + ), + tool_parser=None, + chat_template_kwargs={}, + engine_prompt={"prompt": "Hello"}, + prompt_token_ids=[1], + guided_decoding=None, + uses_dynamo_json_tool_call_fallback=False, + ) + ), + ) + monkeypatch.setattr( + vllm_processor_module.InputProcessor, "assign_request_id", lambda request: None + ) + + def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks): + return SimpleNamespace( + sampling_params=sampling_params.clone(), mm_features=None + ) + + processor = vllm_processor_module.VllmProcessor( + tokenizer=SimpleNamespace(eos_token_id=2, all_special_tokens=[]), + input_processor=SimpleNamespace( + generation_config_fields={}, + renderer=SimpleNamespace( + process_for_engine_async=AsyncMock(return_value={}) + ), + process_inputs=process_inputs, + model_config=None, + ), + output_processor=object(), + tool_parser_class=None, + reasoning_parser_class=None, + routed_engine=object(), + ) + captured = {} + + async def capture_generate_and_stream( + request_id, request, dynamo_preproc, *args, **kwargs + ): + captured.update(dynamo_preproc) + yield {} + + monkeypatch.setattr(processor, "_generate_and_stream", capture_generate_and_stream) + + passthrough = { + "allowed_token_ids": [3, 4], + "bad_words_token_ids": [[7, 8]], + "logprob_token_ids": [5, 6], + "vllm_xargs": {"diffusion_max_steps": 1}, + } + async for _ in processor._generator_inner( + { + "model": "test", + "messages": [{"role": "user", "content": "Hello"}], + **passthrough, + } + ): + pass + + assert captured["extra_args"]["sampling_options"] == passthrough + # Explicit logprob_token_ids replace the top-k count, as on the worker. + assert captured["output_options"]["logprobs"] is None + + @pytest.mark.asyncio async def test_generator_inner_forwards_reasoning_parser_and_model_config( vllm_processor_module, diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 2c387096936e..41d8079c7faf 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -682,6 +682,27 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None: image_url["detail"] = "auto" +# Request fields the vLLM worker applies from extra_args.sampling_options, sent +# as the client wrote them. Keep in step with the Rust preprocessor's +# `sampling_passthrough_args` (lib/llm/src/preprocessor.rs). +_SAMPLING_PASSTHROUGH_FIELDS = ( + "detokenize", + "allowed_token_ids", + "bad_words_token_ids", + "logprob_token_ids", + "vllm_xargs", +) + + +def _sampling_passthrough_args(request: dict[str, Any]) -> dict[str, Any]: + """The request's passthrough fields for the worker's sampling options.""" + return { + key: request[key] + for key in _SAMPLING_PASSTHROUGH_FIELDS + if request.get(key) is not None + } + + def _disabled_as_unset(value: Any, requested: Any) -> Any: """Send a disabled top_k or min_p as unset unless the client disabled it. @@ -1014,6 +1035,10 @@ async def _generator_inner( request_for_sampling.logprobs, getattr(request_for_sampling, "top_logprobs", None), ) + # Explicit logprob_token_ids set the width, so vLLM's verify rejects a + # top-k count next to them; the worker nulls it the same way. + if getattr(sampling_params, "logprob_token_ids", None): + sampling_params.logprobs = None # Response metadata is opt-in, even if generation_config enables it. if request.get("prompt_logprobs") is None or ( request.get("stream") is True @@ -1098,6 +1123,10 @@ async def _generator_inner( } if guided_decoding is not None: dynamo_preproc["sampling_options"]["guided_decoding"] = guided_decoding + if sampling_passthrough := _sampling_passthrough_args(request): + dynamo_preproc.setdefault("extra_args", {})[ + "sampling_options" + ] = sampling_passthrough if reasoning_metadata.engine_reasoning_ended is not None: dynamo_preproc[ "reasoning_ended" diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md index df4e18af7788..fd88892586a9 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md @@ -149,6 +149,10 @@ The field traces through the four phases as follows: and checks that the field exists, so a backend upgrade fails visibly instead of silently dropping the constraint. A mapping test that inspects the sampling parameters proves delivery. A behavioral test that generates tokens under the constraint is still needed to prove the effect. + A Python chat processor (`--dyn-chat-processor vllm` or `sglang`) replaces the Rust + preprocessor, so it is a second delivery path: it receives the named passthrough fields as + top-level keys of its request, and the vLLM processor copies this one to the same + `extra_args.sampling_options` key. 4. **Return the outcome:** The option adds no dedicated response field. Generated text is the client-visible result, while validation or backend compatibility failures surface as errors. Streaming and non-streaming responses otherwise retain their normal shapes. diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index f6a279f16ef2..2280e64476c2 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -995,7 +995,9 @@ are disabled by default. Truthy values are `1`, `true`, `yes`, or `on`, case-ins When truthy, `/v1/chat/completions` and `/v1/completions` ignore and drop unknown top-level request fields instead of rejecting the request. This does not forward arbitrary fields to a - backend. Named passthrough fields remain subject to their normal validation and handling. + backend. Named passthrough fields (`cache_salt`, `stop_token_ids`, `detokenize`, + `allowed_token_ids`, `bad_words_token_ids`, `logprob_token_ids`, `vllm_xargs`) remain subject to + their normal validation and handling. diff --git a/lib/bindings/python/rust/engine.rs b/lib/bindings/python/rust/engine.rs index 8b7434bafb76..5eaffc546c48 100644 --- a/lib/bindings/python/rust/engine.rs +++ b/lib/bindings/python/rust/engine.rs @@ -264,7 +264,15 @@ impl move |py, request| { // Ordinary Chat Completions retain direct typed serialization. if request.responses_tools.is_none() { - return Ok(pythonize(py, &request)?.unbind()); + let object = pythonize(py, &request)?; + // Named passthrough fields (`stop_token_ids`, ...) are not + // serialized with the request; add them at the top level, as + // the client sent them. + let dict = object.downcast::()?; + for (key, value) in request.passthrough_extra_fields() { + dict.set_item(key, pythonize(py, value)?)?; + } + return Ok(object.unbind()); } let value = request .python_chat_processor_request_json(&runtime_config) diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 19c61f941694..5237bd1d0135 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -159,6 +159,15 @@ pub struct NvCreateChatCompletionRequest { } impl NvCreateChatCompletionRequest { + /// The named passthrough fields (`validate::PASSTHROUGH_EXTRA_FIELDS`) this + /// request carries. They are not serialized with the request, so a Python + /// chat processor receives them through this accessor. + pub fn passthrough_extra_fields( + &self, + ) -> impl Iterator { + validate::passthrough_extra_fields(&self.unsupported_fields) + } + /// Resolve the request's reasoning controls into `chat_template_args`. /// Runs once at the HTTP boundary, so every render path reads one answer. /// Model-specific overrides still apply later in the default preprocessor. diff --git a/lib/llm/src/protocols/openai/validate.rs b/lib/llm/src/protocols/openai/validate.rs index 03cedf45428f..36e8080326c1 100644 --- a/lib/llm/src/protocols/openai/validate.rs +++ b/lib/llm/src/protocols/openai/validate.rs @@ -108,6 +108,17 @@ where Ok(fields) } +/// The named passthrough fields present in a request's extra fields. +/// +/// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped. +pub fn passthrough_extra_fields( + extra_fields: &std::collections::HashMap, +) -> impl Iterator { + PASSTHROUGH_EXTRA_FIELDS + .iter() + .filter_map(|key| extra_fields.get(*key).map(|value| (*key, value))) +} + static IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS: LazyLock = LazyLock::new(|| env_is_truthy(DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS)); @@ -1305,6 +1316,24 @@ mod tests { validate_no_unsupported_fields_with_ignore(&unknown_fields(), true).unwrap(); } + #[test] + fn passthrough_extra_fields_skips_ignored_unknown_fields() { + let extra_fields = HashMap::from([ + ("experimental_field".to_string(), json!("value")), + ("stop_token_ids".to_string(), json!([7])), + ("allowed_token_ids".to_string(), json!([10, 11])), + ]); + + let fields: Vec<_> = passthrough_extra_fields(&extra_fields).collect(); + assert_eq!( + fields, + vec![ + ("stop_token_ids", &json!([7])), + ("allowed_token_ids", &json!([10, 11])), + ] + ); + } + #[test] fn validate_no_unsupported_fields_still_validates_passthrough_fields_when_ignoring_unknowns() { let unsupported_fields = HashMap::from([ From b0d68687712950f123a6b9493919443f60fb918a Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 14:09:29 -0800 Subject: [PATCH 2/3] fix(frontend): list explicit logprob token scores; drop detokenize forwarding With logprob_token_ids, each response logprob entry now lists every token the worker scored, as vLLM's chat server does, instead of being cut to top_logprobs (0 when omitted). The vLLM processor no longer forwards detokenize: the worker's token path always turns it off. Fold the passthrough accessor into one method on NvCreateChatCompletionRequest, inline the single-use helper, fold the forwarding test into the logprobs-count test, and drop the unrelated field list from the frontend configuration reference. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../tests/test_vllm_processor_unit.py | 121 +++++------------- .../src/dynamo/frontend/vllm_processor.py | 46 ++++--- .../components/frontend-configuration.mdx | 4 +- .../src/protocols/openai/chat_completions.rs | 29 ++++- lib/llm/src/protocols/openai/validate.rs | 29 ----- 5 files changed, 86 insertions(+), 143 deletions(-) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 60d3d5ade4e5..cc0d989a7a96 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -1441,6 +1441,12 @@ async def stop_at_preprocess(*args, **kwargs): ({"logprobs": True, "top_logprobs": 1}, 1, None), ({"logprobs": True, "top_logprobs": 0, "prompt_logprobs": 0}, 0, 0), ({"logprobs": True, "top_logprobs": 1, "prompt_logprobs": 3}, 1, 3), + # Explicit ids replace the top-k count, as on the worker. + ( + {"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]}, + None, + None, + ), ({"prompt_logprobs": 0}, None, 0), ({"stream": True}, None, None), ({"stream": True, "prompt_logprobs": 0}, None, None), @@ -1477,7 +1483,7 @@ async def test_generator_forwards_chat_logprobs_count( generation_config, ): class RequestForSampling(SimpleNamespace): - model_fields = frozenset({"prompt_logprobs"}) + model_fields = frozenset({"prompt_logprobs", "logprob_token_ids"}) monkeypatch.setattr( vllm_processor_module, @@ -1490,6 +1496,7 @@ class RequestForSampling(SimpleNamespace): logprobs=request_options.get("logprobs"), top_logprobs=request_options.get("top_logprobs"), prompt_logprobs=request_options.get("prompt_logprobs"), + logprob_token_ids=request_options.get("logprob_token_ids"), top_k=None, min_p=None, cache_salt=None, @@ -1559,6 +1566,12 @@ async def capture_generate_and_stream(*args, **kwargs): captured["dynamo_preproc"]["output_options"]["prompt_logprobs"] == expected_prompt_logprobs ) + sampling_passthrough = ( + captured["dynamo_preproc"].get("extra_args", {}).get("sampling_options", {}) + ) + assert sampling_passthrough.get("logprob_token_ids") == request_options.get( + "logprob_token_ids" + ) @pytest.mark.asyncio @@ -1827,95 +1840,6 @@ async def capture_generate_and_stream( assert captured["sampling_options"]["min_p"] == expected_min_p -@pytest.mark.asyncio -async def test_generator_forwards_sampling_passthrough_fields( - vllm_processor_module, - monkeypatch, -): - """Passthrough fields reach the worker as the Rust preprocessor sends them.""" - - class RequestForSampling(SimpleNamespace): - model_fields = frozenset({"logprob_token_ids"}) - - monkeypatch.setattr( - vllm_processor_module, - "preprocess_chat_request", - AsyncMock( - return_value=SimpleNamespace( - request_for_sampling=RequestForSampling( - max_completion_tokens=None, - max_tokens=1, - logprobs=True, - top_logprobs=0, - cache_salt=None, - mm_processor_kwargs=None, - top_k=None, - min_p=None, - logprob_token_ids=[5, 6], - ), - tool_parser=None, - chat_template_kwargs={}, - engine_prompt={"prompt": "Hello"}, - prompt_token_ids=[1], - guided_decoding=None, - uses_dynamo_json_tool_call_fallback=False, - ) - ), - ) - monkeypatch.setattr( - vllm_processor_module.InputProcessor, "assign_request_id", lambda request: None - ) - - def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks): - return SimpleNamespace( - sampling_params=sampling_params.clone(), mm_features=None - ) - - processor = vllm_processor_module.VllmProcessor( - tokenizer=SimpleNamespace(eos_token_id=2, all_special_tokens=[]), - input_processor=SimpleNamespace( - generation_config_fields={}, - renderer=SimpleNamespace( - process_for_engine_async=AsyncMock(return_value={}) - ), - process_inputs=process_inputs, - model_config=None, - ), - output_processor=object(), - tool_parser_class=None, - reasoning_parser_class=None, - routed_engine=object(), - ) - captured = {} - - async def capture_generate_and_stream( - request_id, request, dynamo_preproc, *args, **kwargs - ): - captured.update(dynamo_preproc) - yield {} - - monkeypatch.setattr(processor, "_generate_and_stream", capture_generate_and_stream) - - passthrough = { - "allowed_token_ids": [3, 4], - "bad_words_token_ids": [[7, 8]], - "logprob_token_ids": [5, 6], - "vllm_xargs": {"diffusion_max_steps": 1}, - } - async for _ in processor._generator_inner( - { - "model": "test", - "messages": [{"role": "user", "content": "Hello"}], - **passthrough, - } - ): - pass - - assert captured["extra_args"]["sampling_options"] == passthrough - # Explicit logprob_token_ids replace the top-k count, as on the worker. - assert captured["output_options"]["logprobs"] is None - - @pytest.mark.asyncio async def test_generator_inner_forwards_reasoning_parser_and_model_config( vllm_processor_module, @@ -5049,6 +4973,23 @@ def test_sampling_logprobs_count_accepts_chat_bool(vllm_processor_module): assert count(False, True) is None +@pytest.mark.parametrize( + "request_options,expected", + [ + ({"logprobs": True}, 0), + ({"logprobs": True, "top_logprobs": 1}, 1), + ({"logprobs": True, "logprob_token_ids": [5, 6]}, 3), + ({"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]}, 3), + ({"logprob_token_ids": [5, 6]}, None), + ], +) +def test_choice_top_logprobs_lists_every_explicit_token( + vllm_processor_module, request_options, expected +): + """Explicit ids list the sampled token and every named id, as vLLM does.""" + assert vllm_processor_module._choice_top_logprobs(request_options) == expected + + def test_chat_choice_logprobs_from_worker_chunk(vllm_processor_module, tokenizer): built = vllm_processor_module._chat_choice_logprobs( [ diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 41d8079c7faf..4240d62ad260 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -97,6 +97,21 @@ def _sampling_logprobs_count(logprobs: Any, top_logprobs: Any) -> int | None: return None +def _choice_top_logprobs(request: dict[str, Any]) -> int | None: + """How many alternatives each response logprob entry lists. + + With explicit ``logprob_token_ids`` the worker scores the sampled token and + each named id, and vLLM's chat server lists them all, whatever + ``top_logprobs`` says. + """ + count = _sampling_logprobs_count( + request.get("logprobs"), request.get("top_logprobs") + ) + if count is not None and (token_ids := request.get("logprob_token_ids")): + return len(token_ids) + 1 + return count + + def _reject_unsupported_chat_logprobs(logprobs: Any, top_logprobs: Any) -> None: """Validate chat logprob options before preprocessing. @@ -683,10 +698,10 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None: # Request fields the vLLM worker applies from extra_args.sampling_options, sent -# as the client wrote them. Keep in step with the Rust preprocessor's -# `sampling_passthrough_args` (lib/llm/src/preprocessor.rs). +# as the client wrote them: the Rust preprocessor's `sampling_passthrough_args` +# (lib/llm/src/preprocessor.rs) without `detokenize`, which the worker's token +# path always turns off. _SAMPLING_PASSTHROUGH_FIELDS = ( - "detokenize", "allowed_token_ids", "bad_words_token_ids", "logprob_token_ids", @@ -694,15 +709,6 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None: ) -def _sampling_passthrough_args(request: dict[str, Any]) -> dict[str, Any]: - """The request's passthrough fields for the worker's sampling options.""" - return { - key: request[key] - for key in _SAMPLING_PASSTHROUGH_FIELDS - if request.get(key) is not None - } - - def _disabled_as_unset(value: Any, requested: Any) -> Any: """Send a disabled top_k or min_p as unset unless the client disabled it. @@ -1123,10 +1129,13 @@ async def _generator_inner( } if guided_decoding is not None: dynamo_preproc["sampling_options"]["guided_decoding"] = guided_decoding - if sampling_passthrough := _sampling_passthrough_args(request): - dynamo_preproc.setdefault("extra_args", {})[ - "sampling_options" - ] = sampling_passthrough + sampling_passthrough = { + key: request[key] + for key in _SAMPLING_PASSTHROUGH_FIELDS + if request.get(key) is not None + } + if sampling_passthrough: + dynamo_preproc["extra_args"] = {"sampling_options": sampling_passthrough} if reasoning_metadata.engine_reasoning_ended is not None: dynamo_preproc[ "reasoning_ended" @@ -1298,10 +1307,7 @@ async def _generate_and_stream( audio_count = len(_mm_counts.get("audio_url", [])) pending_choice_logprobs: dict[int, list[dict[str, Any]]] = {} emitted_choice_tokens: dict[int, list[int]] = {} - choice_top_logprobs = _sampling_logprobs_count( - request.get("logprobs"), - request.get("top_logprobs"), - ) + choice_top_logprobs = _choice_top_logprobs(request) return_tokens_as_token_ids = request.get("return_tokens_as_token_ids") is True prompt_logprobs = PromptLogprobsAccumulator(request) pending_terminals: dict[int, dict[str, Any]] = {} diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 2280e64476c2..f6a279f16ef2 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -995,9 +995,7 @@ are disabled by default. Truthy values are `1`, `true`, `yes`, or `on`, case-ins When truthy, `/v1/chat/completions` and `/v1/completions` ignore and drop unknown top-level request fields instead of rejecting the request. This does not forward arbitrary fields to a - backend. Named passthrough fields (`cache_salt`, `stop_token_ids`, `detokenize`, - `allowed_token_ids`, `bad_words_token_ids`, `logprob_token_ids`, `vllm_xargs`) remain subject to - their normal validation and handling. + backend. Named passthrough fields remain subject to their normal validation and handling. diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 5237bd1d0135..2951f1bdd835 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -162,10 +162,13 @@ impl NvCreateChatCompletionRequest { /// The named passthrough fields (`validate::PASSTHROUGH_EXTRA_FIELDS`) this /// request carries. They are not serialized with the request, so a Python /// chat processor receives them through this accessor. + /// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped. pub fn passthrough_extra_fields( &self, ) -> impl Iterator { - validate::passthrough_extra_fields(&self.unsupported_fields) + validate::PASSTHROUGH_EXTRA_FIELDS + .iter() + .filter_map(|key| self.unsupported_fields.get(*key).map(|value| (*key, value))) } /// Resolve the request's reasoning controls into `chat_template_args`. @@ -1176,6 +1179,30 @@ mod tests { assert!(err.to_string().contains("stop_token_ids")); } + #[test] + fn passthrough_extra_fields_skips_ignored_unknown_fields() { + let mut request: NvCreateChatCompletionRequest = serde_json::from_value(json!({ + "model": "test-model", + "messages": [{"role": "user", "content": "Hello"}], + "stop_token_ids": [7], + "allowed_token_ids": [10, 11], + })) + .expect("Failed to deserialize request"); + // DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS keeps unknown fields in the map. + request + .unsupported_fields + .insert("experimental_field".to_string(), json!("value")); + + let fields: Vec<_> = request.passthrough_extra_fields().collect(); + assert_eq!( + fields, + vec![ + ("stop_token_ids", &json!([7])), + ("allowed_token_ids", &json!([10, 11])), + ] + ); + } + #[test] fn test_stop_sequence_limit_enforced_consistently() { use crate::protocols::openai::validate::MAX_STOP_SEQUENCES; diff --git a/lib/llm/src/protocols/openai/validate.rs b/lib/llm/src/protocols/openai/validate.rs index 36e8080326c1..03cedf45428f 100644 --- a/lib/llm/src/protocols/openai/validate.rs +++ b/lib/llm/src/protocols/openai/validate.rs @@ -108,17 +108,6 @@ where Ok(fields) } -/// The named passthrough fields present in a request's extra fields. -/// -/// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped. -pub fn passthrough_extra_fields( - extra_fields: &std::collections::HashMap, -) -> impl Iterator { - PASSTHROUGH_EXTRA_FIELDS - .iter() - .filter_map(|key| extra_fields.get(*key).map(|value| (*key, value))) -} - static IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS: LazyLock = LazyLock::new(|| env_is_truthy(DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS)); @@ -1316,24 +1305,6 @@ mod tests { validate_no_unsupported_fields_with_ignore(&unknown_fields(), true).unwrap(); } - #[test] - fn passthrough_extra_fields_skips_ignored_unknown_fields() { - let extra_fields = HashMap::from([ - ("experimental_field".to_string(), json!("value")), - ("stop_token_ids".to_string(), json!([7])), - ("allowed_token_ids".to_string(), json!([10, 11])), - ]); - - let fields: Vec<_> = passthrough_extra_fields(&extra_fields).collect(); - assert_eq!( - fields, - vec![ - ("stop_token_ids", &json!([7])), - ("allowed_token_ids", &json!([10, 11])), - ] - ); - } - #[test] fn validate_no_unsupported_fields_still_validates_passthrough_fields_when_ignoring_unknowns() { let unsupported_fields = HashMap::from([ From 5b99ace4727533274e2014c5ee1db4d0e37e8cbe Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 15:37:40 -0800 Subject: [PATCH 3/3] docs(frontend): name only the vLLM processor as a bad_words_token_ids path Only the vLLM worker reads extra_args.sampling_options, so the SGLang processor is not a delivery path for bad_words_token_ids. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../modular-components/frontend/protocol-field-handling.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md index fd88892586a9..b33adcc49daa 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md @@ -149,10 +149,9 @@ The field traces through the four phases as follows: and checks that the field exists, so a backend upgrade fails visibly instead of silently dropping the constraint. A mapping test that inspects the sampling parameters proves delivery. A behavioral test that generates tokens under the constraint is still needed to prove the effect. - A Python chat processor (`--dyn-chat-processor vllm` or `sglang`) replaces the Rust - preprocessor, so it is a second delivery path: it receives the named passthrough fields as - top-level keys of its request, and the vLLM processor copies this one to the same - `extra_args.sampling_options` key. + The vLLM Python chat processor (`--dyn-chat-processor vllm`) replaces the Rust preprocessor, + so it is a second delivery path: it receives the named passthrough fields as top-level keys of + its request and copies this one to the same `extra_args.sampling_options` key. 4. **Return the outcome:** The option adds no dedicated response field. Generated text is the client-visible result, while validation or backend compatibility failures surface as errors. Streaming and non-streaming responses otherwise retain their normal shapes.