Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1441,6 +1441,12 @@ async def stop_at_preprocess(*args, **kwargs):
({"logprobs": True, "top_logprobs": 1}, 1, None),
({"logprobs": True, "top_logprobs": 0, "prompt_logprobs": 0}, 0, 0),
({"logprobs": True, "top_logprobs": 1, "prompt_logprobs": 3}, 1, 3),
# Explicit ids replace the top-k count, as on the worker.
(
{"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]},
None,
None,
),
({"prompt_logprobs": 0}, None, 0),
({"stream": True}, None, None),
({"stream": True, "prompt_logprobs": 0}, None, None),
Expand Down Expand Up @@ -1477,7 +1483,7 @@ async def test_generator_forwards_chat_logprobs_count(
generation_config,
):
class RequestForSampling(SimpleNamespace):
model_fields = frozenset({"prompt_logprobs"})
model_fields = frozenset({"prompt_logprobs", "logprob_token_ids"})

monkeypatch.setattr(
vllm_processor_module,
Expand All @@ -1490,6 +1496,7 @@ class RequestForSampling(SimpleNamespace):
logprobs=request_options.get("logprobs"),
top_logprobs=request_options.get("top_logprobs"),
prompt_logprobs=request_options.get("prompt_logprobs"),
logprob_token_ids=request_options.get("logprob_token_ids"),
top_k=None,
min_p=None,
cache_salt=None,
Expand Down Expand Up @@ -1559,6 +1566,12 @@ async def capture_generate_and_stream(*args, **kwargs):
captured["dynamo_preproc"]["output_options"]["prompt_logprobs"]
== expected_prompt_logprobs
)
sampling_passthrough = (
captured["dynamo_preproc"].get("extra_args", {}).get("sampling_options", {})
)
assert sampling_passthrough.get("logprob_token_ids") == request_options.get(
"logprob_token_ids"
)


@pytest.mark.asyncio
Expand Down Expand Up @@ -4960,6 +4973,23 @@ def test_sampling_logprobs_count_accepts_chat_bool(vllm_processor_module):
assert count(False, True) is None


@pytest.mark.parametrize(
"request_options,expected",
[
({"logprobs": True}, 0),
({"logprobs": True, "top_logprobs": 1}, 1),
({"logprobs": True, "logprob_token_ids": [5, 6]}, 3),
({"logprobs": True, "top_logprobs": 1, "logprob_token_ids": [5, 6]}, 3),
({"logprob_token_ids": [5, 6]}, None),
],
)
def test_choice_top_logprobs_lists_every_explicit_token(
vllm_processor_module, request_options, expected
):
"""Explicit ids list the sampled token and every named id, as vLLM does."""
assert vllm_processor_module._choice_top_logprobs(request_options) == expected


def test_chat_choice_logprobs_from_worker_chunk(vllm_processor_module, tokenizer):
built = vllm_processor_module._chat_choice_logprobs(
[
Expand Down
43 changes: 39 additions & 4 deletions components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,21 @@ def _sampling_logprobs_count(logprobs: Any, top_logprobs: Any) -> int | None:
return None


def _choice_top_logprobs(request: dict[str, Any]) -> int | None:
"""How many alternatives each response logprob entry lists.

With explicit ``logprob_token_ids`` the worker scores the sampled token and
each named id, and vLLM's chat server lists them all, whatever
``top_logprobs`` says.
"""
count = _sampling_logprobs_count(
request.get("logprobs"), request.get("top_logprobs")
)
if count is not None and (token_ids := request.get("logprob_token_ids")):
return len(token_ids) + 1
return count


def _reject_unsupported_chat_logprobs(logprobs: Any, top_logprobs: Any) -> None:
"""Validate chat logprob options before preprocessing.

Expand Down Expand Up @@ -682,6 +697,18 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None:
image_url["detail"] = "auto"


# Request fields the vLLM worker applies from extra_args.sampling_options, sent
# as the client wrote them: the Rust preprocessor's `sampling_passthrough_args`
# (lib/llm/src/preprocessor.rs) without `detokenize`, which the worker's token
# path always turns off.
_SAMPLING_PASSTHROUGH_FIELDS = (
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
"allowed_token_ids",
"bad_words_token_ids",
"logprob_token_ids",
"vllm_xargs",
)


def _disabled_as_unset(value: Any, requested: Any) -> Any:
"""Send a disabled top_k or min_p as unset unless the client disabled it.

Expand Down Expand Up @@ -1014,6 +1041,10 @@ async def _generator_inner(
request_for_sampling.logprobs,
getattr(request_for_sampling, "top_logprobs", None),
)
# Explicit logprob_token_ids set the width, so vLLM's verify rejects a
# top-k count next to them; the worker nulls it the same way.
if getattr(sampling_params, "logprob_token_ids", None):
sampling_params.logprobs = None
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
# Response metadata is opt-in, even if generation_config enables it.
if request.get("prompt_logprobs") is None or (
request.get("stream") is True
Expand Down Expand Up @@ -1098,6 +1129,13 @@ async def _generator_inner(
}
if guided_decoding is not None:
dynamo_preproc["sampling_options"]["guided_decoding"] = guided_decoding
sampling_passthrough = {
key: request[key]
for key in _SAMPLING_PASSTHROUGH_FIELDS
if request.get(key) is not None
}
if sampling_passthrough:
dynamo_preproc["extra_args"] = {"sampling_options": sampling_passthrough}
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
if reasoning_metadata.engine_reasoning_ended is not None:
dynamo_preproc[
"reasoning_ended"
Expand Down Expand Up @@ -1269,10 +1307,7 @@ async def _generate_and_stream(
audio_count = len(_mm_counts.get("audio_url", []))
pending_choice_logprobs: dict[int, list[dict[str, Any]]] = {}
emitted_choice_tokens: dict[int, list[int]] = {}
choice_top_logprobs = _sampling_logprobs_count(
request.get("logprobs"),
request.get("top_logprobs"),
)
choice_top_logprobs = _choice_top_logprobs(request)
return_tokens_as_token_ids = request.get("return_tokens_as_token_ids") is True
prompt_logprobs = PromptLogprobsAccumulator(request)
pending_terminals: dict[int, dict[str, Any]] = {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ The field traces through the four phases as follows:
and checks that the field exists, so a backend upgrade fails visibly instead of silently dropping
the constraint. A mapping test that inspects the sampling parameters proves delivery. A
behavioral test that generates tokens under the constraint is still needed to prove the effect.
A Python chat processor (`--dyn-chat-processor vllm` or `sglang`) replaces the Rust
preprocessor, so it is a second delivery path: it receives the named passthrough fields as
top-level keys of its request, and the vLLM processor copies this one to the same
`extra_args.sampling_options` key.
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
4. **Return the outcome:** The option adds no dedicated response field. Generated text is the
client-visible result, while validation or backend compatibility failures surface as errors.
Streaming and non-streaming responses otherwise retain their normal shapes.
Expand Down
10 changes: 9 additions & 1 deletion lib/bindings/python/rust/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,15 @@ impl
move |py, request| {
// Ordinary Chat Completions retain direct typed serialization.
if request.responses_tools.is_none() {
return Ok(pythonize(py, &request)?.unbind());
let object = pythonize(py, &request)?;
// Named passthrough fields (`stop_token_ids`, ...) are not
// serialized with the request; add them at the top level, as
// the client sent them.
let dict = object.downcast::<PyDict>()?;
for (key, value) in request.passthrough_extra_fields() {
dict.set_item(key, pythonize(py, value)?)?;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
return Ok(object.unbind());
}
let value = request
.python_chat_processor_request_json(&runtime_config)
Expand Down
36 changes: 36 additions & 0 deletions lib/llm/src/protocols/openai/chat_completions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,18 @@ pub struct NvCreateChatCompletionRequest {
}

impl NvCreateChatCompletionRequest {
/// The named passthrough fields (`validate::PASSTHROUGH_EXTRA_FIELDS`) this
/// request carries. They are not serialized with the request, so a Python
/// chat processor receives them through this accessor.
/// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped.
pub fn passthrough_extra_fields(
&self,
) -> impl Iterator<Item = (&'static str, &serde_json::Value)> {
validate::PASSTHROUGH_EXTRA_FIELDS
.iter()
.filter_map(|key| self.unsupported_fields.get(*key).map(|value| (*key, value)))
}

/// Resolve the request's reasoning controls into `chat_template_args`.
/// Runs once at the HTTP boundary, so every render path reads one answer.
/// Model-specific overrides still apply later in the default preprocessor.
Expand Down Expand Up @@ -1167,6 +1179,30 @@ mod tests {
assert!(err.to_string().contains("stop_token_ids"));
}

#[test]
fn passthrough_extra_fields_skips_ignored_unknown_fields() {
let mut request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test-model",
"messages": [{"role": "user", "content": "Hello"}],
"stop_token_ids": [7],
"allowed_token_ids": [10, 11],
}))
.expect("Failed to deserialize request");
// DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS keeps unknown fields in the map.
request
.unsupported_fields
.insert("experimental_field".to_string(), json!("value"));

let fields: Vec<_> = request.passthrough_extra_fields().collect();
assert_eq!(
fields,
vec![
("stop_token_ids", &json!([7])),
("allowed_token_ids", &json!([10, 11])),
]
);
}

#[test]
fn test_stop_sequence_limit_enforced_consistently() {
use crate::protocols::openai::validate::MAX_STOP_SEQUENCES;
Expand Down
Loading