diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index a36fe75e5e2d..cc0d989a7a96 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -1441,6 +1441,12 @@ async def stop_at_preprocess(*args, **kwargs): ({"logprobs": True, "top_logprobs": 1}, 1, None), ({"logprobs": True, "top_logprobs": 0, "prompt_logprobs": 0}, 0, 0), ({"logprobs": True, "top_logprobs": 1, "prompt_logprobs": 3}, 1, 3), + # Explicit ids replace the top-k count, as on the worker. + ( + {"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]}, + None, + None, + ), ({"prompt_logprobs": 0}, None, 0), ({"stream": True}, None, None), ({"stream": True, "prompt_logprobs": 0}, None, None), @@ -1477,7 +1483,7 @@ async def test_generator_forwards_chat_logprobs_count( generation_config, ): class RequestForSampling(SimpleNamespace): - model_fields = frozenset({"prompt_logprobs"}) + model_fields = frozenset({"prompt_logprobs", "logprob_token_ids"}) monkeypatch.setattr( vllm_processor_module, @@ -1490,6 +1496,7 @@ class RequestForSampling(SimpleNamespace): logprobs=request_options.get("logprobs"), top_logprobs=request_options.get("top_logprobs"), prompt_logprobs=request_options.get("prompt_logprobs"), + logprob_token_ids=request_options.get("logprob_token_ids"), top_k=None, min_p=None, cache_salt=None, @@ -1559,6 +1566,12 @@ async def capture_generate_and_stream(*args, **kwargs): captured["dynamo_preproc"]["output_options"]["prompt_logprobs"] == expected_prompt_logprobs ) + sampling_passthrough = ( + captured["dynamo_preproc"].get("extra_args", {}).get("sampling_options", {}) + ) + assert sampling_passthrough.get("logprob_token_ids") == request_options.get( + "logprob_token_ids" + ) @pytest.mark.asyncio @@ -4960,6 +4973,23 @@ def test_sampling_logprobs_count_accepts_chat_bool(vllm_processor_module): assert count(False, True) is None +@pytest.mark.parametrize( + "request_options,expected", + [ + ({"logprobs": True}, 0), + ({"logprobs": True, "top_logprobs": 1}, 1), + ({"logprobs": True, "logprob_token_ids": [5, 6]}, 3), + ({"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]}, 3), + ({"logprob_token_ids": [5, 6]}, None), + ], +) +def test_choice_top_logprobs_lists_every_explicit_token( + vllm_processor_module, request_options, expected +): + """Explicit ids list the sampled token and every named id, as vLLM does.""" + assert vllm_processor_module._choice_top_logprobs(request_options) == expected + + def test_chat_choice_logprobs_from_worker_chunk(vllm_processor_module, tokenizer): built = vllm_processor_module._chat_choice_logprobs( [ diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 2c387096936e..4240d62ad260 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -97,6 +97,21 @@ def _sampling_logprobs_count(logprobs: Any, top_logprobs: Any) -> int | None: return None +def _choice_top_logprobs(request: dict[str, Any]) -> int | None: + """How many alternatives each response logprob entry lists. + + With explicit ``logprob_token_ids`` the worker scores the sampled token and + each named id, and vLLM's chat server lists them all, whatever + ``top_logprobs`` says. + """ + count = _sampling_logprobs_count( + request.get("logprobs"), request.get("top_logprobs") + ) + if count is not None and (token_ids := request.get("logprob_token_ids")): + return len(token_ids) + 1 + return count + + def _reject_unsupported_chat_logprobs(logprobs: Any, top_logprobs: Any) -> None: """Validate chat logprob options before preprocessing. @@ -682,6 +697,18 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None: image_url["detail"] = "auto" +# Request fields the vLLM worker applies from extra_args.sampling_options, sent +# as the client wrote them: the Rust preprocessor's `sampling_passthrough_args` +# (lib/llm/src/preprocessor.rs) without `detokenize`, which the worker's token +# path always turns off. +_SAMPLING_PASSTHROUGH_FIELDS = ( + "allowed_token_ids", + "bad_words_token_ids", + "logprob_token_ids", + "vllm_xargs", +) + + def _disabled_as_unset(value: Any, requested: Any) -> Any: """Send a disabled top_k or min_p as unset unless the client disabled it. @@ -1014,6 +1041,10 @@ async def _generator_inner( request_for_sampling.logprobs, getattr(request_for_sampling, "top_logprobs", None), ) + # Explicit logprob_token_ids set the width, so vLLM's verify rejects a + # top-k count next to them; the worker nulls it the same way. + if getattr(sampling_params, "logprob_token_ids", None): + sampling_params.logprobs = None # Response metadata is opt-in, even if generation_config enables it. if request.get("prompt_logprobs") is None or ( request.get("stream") is True @@ -1098,6 +1129,13 @@ async def _generator_inner( } if guided_decoding is not None: dynamo_preproc["sampling_options"]["guided_decoding"] = guided_decoding + sampling_passthrough = { + key: request[key] + for key in _SAMPLING_PASSTHROUGH_FIELDS + if request.get(key) is not None + } + if sampling_passthrough: + dynamo_preproc["extra_args"] = {"sampling_options": sampling_passthrough} if reasoning_metadata.engine_reasoning_ended is not None: dynamo_preproc[ "reasoning_ended" @@ -1269,10 +1307,7 @@ async def _generate_and_stream( audio_count = len(_mm_counts.get("audio_url", [])) pending_choice_logprobs: dict[int, list[dict[str, Any]]] = {} emitted_choice_tokens: dict[int, list[int]] = {} - choice_top_logprobs = _sampling_logprobs_count( - request.get("logprobs"), - request.get("top_logprobs"), - ) + choice_top_logprobs = _choice_top_logprobs(request) return_tokens_as_token_ids = request.get("return_tokens_as_token_ids") is True prompt_logprobs = PromptLogprobsAccumulator(request) pending_terminals: dict[int, dict[str, Any]] = {} diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md index df4e18af7788..b33adcc49daa 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/protocol-field-handling.md @@ -149,6 +149,9 @@ The field traces through the four phases as follows: and checks that the field exists, so a backend upgrade fails visibly instead of silently dropping the constraint. A mapping test that inspects the sampling parameters proves delivery. A behavioral test that generates tokens under the constraint is still needed to prove the effect. + The vLLM Python chat processor (`--dyn-chat-processor vllm`) replaces the Rust preprocessor, + so it is a second delivery path: it receives the named passthrough fields as top-level keys of + its request and copies this one to the same `extra_args.sampling_options` key. 4. **Return the outcome:** The option adds no dedicated response field. Generated text is the client-visible result, while validation or backend compatibility failures surface as errors. Streaming and non-streaming responses otherwise retain their normal shapes. diff --git a/lib/bindings/python/rust/engine.rs b/lib/bindings/python/rust/engine.rs index 8b7434bafb76..5eaffc546c48 100644 --- a/lib/bindings/python/rust/engine.rs +++ b/lib/bindings/python/rust/engine.rs @@ -264,7 +264,15 @@ impl move |py, request| { // Ordinary Chat Completions retain direct typed serialization. if request.responses_tools.is_none() { - return Ok(pythonize(py, &request)?.unbind()); + let object = pythonize(py, &request)?; + // Named passthrough fields (`stop_token_ids`, ...) are not + // serialized with the request; add them at the top level, as + // the client sent them. + let dict = object.downcast::()?; + for (key, value) in request.passthrough_extra_fields() { + dict.set_item(key, pythonize(py, value)?)?; + } + return Ok(object.unbind()); } let value = request .python_chat_processor_request_json(&runtime_config) diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 19c61f941694..2951f1bdd835 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -159,6 +159,18 @@ pub struct NvCreateChatCompletionRequest { } impl NvCreateChatCompletionRequest { + /// The named passthrough fields (`validate::PASSTHROUGH_EXTRA_FIELDS`) this + /// request carries. They are not serialized with the request, so a Python + /// chat processor receives them through this accessor. + /// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped. + pub fn passthrough_extra_fields( + &self, + ) -> impl Iterator { + validate::PASSTHROUGH_EXTRA_FIELDS + .iter() + .filter_map(|key| self.unsupported_fields.get(*key).map(|value| (*key, value))) + } + /// Resolve the request's reasoning controls into `chat_template_args`. /// Runs once at the HTTP boundary, so every render path reads one answer. /// Model-specific overrides still apply later in the default preprocessor. @@ -1167,6 +1179,30 @@ mod tests { assert!(err.to_string().contains("stop_token_ids")); } + #[test] + fn passthrough_extra_fields_skips_ignored_unknown_fields() { + let mut request: NvCreateChatCompletionRequest = serde_json::from_value(json!({ + "model": "test-model", + "messages": [{"role": "user", "content": "Hello"}], + "stop_token_ids": [7], + "allowed_token_ids": [10, 11], + })) + .expect("Failed to deserialize request"); + // DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS keeps unknown fields in the map. + request + .unsupported_fields + .insert("experimental_field".to_string(), json!("value")); + + let fields: Vec<_> = request.passthrough_extra_fields().collect(); + assert_eq!( + fields, + vec![ + ("stop_token_ids", &json!([7])), + ("allowed_token_ids", &json!([10, 11])), + ] + ); + } + #[test] fn test_stop_sequence_limit_enforced_consistently() { use crate::protocols::openai::validate::MAX_STOP_SEQUENCES;