Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1827,6 +1827,95 @@ async def capture_generate_and_stream(
assert captured["sampling_options"]["min_p"] == expected_min_p


@pytest.mark.asyncio
async def test_generator_forwards_sampling_passthrough_fields(
vllm_processor_module,
monkeypatch,
):
"""Passthrough fields reach the worker as the Rust preprocessor sends them."""

class RequestForSampling(SimpleNamespace):
model_fields = frozenset({"logprob_token_ids"})

monkeypatch.setattr(
vllm_processor_module,
"preprocess_chat_request",
AsyncMock(
return_value=SimpleNamespace(
request_for_sampling=RequestForSampling(
max_completion_tokens=None,
max_tokens=1,
logprobs=True,
top_logprobs=0,
cache_salt=None,
mm_processor_kwargs=None,
top_k=None,
min_p=None,
logprob_token_ids=[5, 6],
),
tool_parser=None,
chat_template_kwargs={},
engine_prompt={"prompt": "Hello"},
prompt_token_ids=[1],
guided_decoding=None,
uses_dynamo_json_tool_call_fallback=False,
)
),
)
monkeypatch.setattr(
vllm_processor_module.InputProcessor, "assign_request_id", lambda request: None
)

def process_inputs(request_id, engine_inputs, sampling_params, supported_tasks):
return SimpleNamespace(
sampling_params=sampling_params.clone(), mm_features=None
)

processor = vllm_processor_module.VllmProcessor(
tokenizer=SimpleNamespace(eos_token_id=2, all_special_tokens=[]),
input_processor=SimpleNamespace(
generation_config_fields={},
renderer=SimpleNamespace(
process_for_engine_async=AsyncMock(return_value={})
),
process_inputs=process_inputs,
model_config=None,
),
output_processor=object(),
tool_parser_class=None,
reasoning_parser_class=None,
routed_engine=object(),
)
captured = {}

async def capture_generate_and_stream(
request_id, request, dynamo_preproc, *args, **kwargs
):
captured.update(dynamo_preproc)
yield {}

monkeypatch.setattr(processor, "_generate_and_stream", capture_generate_and_stream)

passthrough = {
"allowed_token_ids": [3, 4],
"bad_words_token_ids": [[7, 8]],
"logprob_token_ids": [5, 6],
"vllm_xargs": {"diffusion_max_steps": 1},
}
async for _ in processor._generator_inner(
{
"model": "test",
"messages": [{"role": "user", "content": "Hello"}],
**passthrough,
}
):
pass

assert captured["extra_args"]["sampling_options"] == passthrough
# Explicit logprob_token_ids replace the top-k count, as on the worker.
assert captured["output_options"]["logprobs"] is None


@pytest.mark.asyncio
async def test_generator_inner_forwards_reasoning_parser_and_model_config(
vllm_processor_module,
Expand Down
29 changes: 29 additions & 0 deletions components/src/dynamo/frontend/vllm_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,27 @@ def _normalize_vllm_image_parts(messages: list[Any]) -> None:
image_url["detail"] = "auto"


# Request fields the vLLM worker applies from extra_args.sampling_options, sent
# as the client wrote them. Keep in step with the Rust preprocessor's
# `sampling_passthrough_args` (lib/llm/src/preprocessor.rs).
_SAMPLING_PASSTHROUGH_FIELDS = (
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
"detokenize",
"allowed_token_ids",
"bad_words_token_ids",
"logprob_token_ids",
"vllm_xargs",
)


def _sampling_passthrough_args(request: dict[str, Any]) -> dict[str, Any]:
"""The request's passthrough fields for the worker's sampling options."""
return {
key: request[key]
for key in _SAMPLING_PASSTHROUGH_FIELDS
if request.get(key) is not None
}


def _disabled_as_unset(value: Any, requested: Any) -> Any:
"""Send a disabled top_k or min_p as unset unless the client disabled it.

Expand Down Expand Up @@ -1014,6 +1035,10 @@ async def _generator_inner(
request_for_sampling.logprobs,
getattr(request_for_sampling, "top_logprobs", None),
)
# Explicit logprob_token_ids set the width, so vLLM's verify rejects a
# top-k count next to them; the worker nulls it the same way.
if getattr(sampling_params, "logprob_token_ids", None):
sampling_params.logprobs = None
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
# Response metadata is opt-in, even if generation_config enables it.
if request.get("prompt_logprobs") is None or (
request.get("stream") is True
Expand Down Expand Up @@ -1098,6 +1123,10 @@ async def _generator_inner(
}
if guided_decoding is not None:
dynamo_preproc["sampling_options"]["guided_decoding"] = guided_decoding
if sampling_passthrough := _sampling_passthrough_args(request):
dynamo_preproc.setdefault("extra_args", {})[
"sampling_options"
] = sampling_passthrough
if reasoning_metadata.engine_reasoning_ended is not None:
dynamo_preproc[
"reasoning_ended"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ The field traces through the four phases as follows:
and checks that the field exists, so a backend upgrade fails visibly instead of silently dropping
the constraint. A mapping test that inspects the sampling parameters proves delivery. A
behavioral test that generates tokens under the constraint is still needed to prove the effect.
A Python chat processor (`--dyn-chat-processor vllm` or `sglang`) replaces the Rust
preprocessor, so it is a second delivery path: it receives the named passthrough fields as
top-level keys of its request, and the vLLM processor copies this one to the same
`extra_args.sampling_options` key.
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
4. **Return the outcome:** The option adds no dedicated response field. Generated text is the
client-visible result, while validation or backend compatibility failures surface as errors.
Streaming and non-streaming responses otherwise retain their normal shapes.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -995,7 +995,9 @@ are disabled by default. Truthy values are `1`, `true`, `yes`, or `on`, case-ins
<ParamField path="DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS" type="boolean" default="false">
When truthy, `/v1/chat/completions` and `/v1/completions` ignore and drop unknown top-level
request fields instead of rejecting the request. This does not forward arbitrary fields to a
backend. Named passthrough fields remain subject to their normal validation and handling.
backend. Named passthrough fields (`cache_salt`, `stop_token_ids`, `detokenize`,
`allowed_token_ids`, `bad_words_token_ids`, `logprob_token_ids`, `vllm_xargs`) remain subject to
their normal validation and handling.
</ParamField>

<ParamField path="DYN_DISABLE_OPENAI_STRICT_SCHEMA_VALIDATION" type="boolean" default="false">
Expand Down
10 changes: 9 additions & 1 deletion lib/bindings/python/rust/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,15 @@ impl
move |py, request| {
// Ordinary Chat Completions retain direct typed serialization.
if request.responses_tools.is_none() {
return Ok(pythonize(py, &request)?.unbind());
let object = pythonize(py, &request)?;
// Named passthrough fields (`stop_token_ids`, ...) are not
// serialized with the request; add them at the top level, as
// the client sent them.
let dict = object.downcast::<PyDict>()?;
for (key, value) in request.passthrough_extra_fields() {
dict.set_item(key, pythonize(py, value)?)?;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
return Ok(object.unbind());
}
let value = request
.python_chat_processor_request_json(&runtime_config)
Expand Down
9 changes: 9 additions & 0 deletions lib/llm/src/protocols/openai/chat_completions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,15 @@ pub struct NvCreateChatCompletionRequest {
}

impl NvCreateChatCompletionRequest {
/// The named passthrough fields (`validate::PASSTHROUGH_EXTRA_FIELDS`) this
/// request carries. They are not serialized with the request, so a Python
/// chat processor receives them through this accessor.
pub fn passthrough_extra_fields(
&self,
) -> impl Iterator<Item = (&'static str, &serde_json::Value)> {
validate::passthrough_extra_fields(&self.unsupported_fields)
}

/// Resolve the request's reasoning controls into `chat_template_args`.
/// Runs once at the HTTP boundary, so every render path reads one answer.
/// Model-specific overrides still apply later in the default preprocessor.
Expand Down
29 changes: 29 additions & 0 deletions lib/llm/src/protocols/openai/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,17 @@ where
Ok(fields)
}

/// The named passthrough fields present in a request's extra fields.
///
/// Unknown fields kept by `DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS` are skipped.
pub fn passthrough_extra_fields(
extra_fields: &std::collections::HashMap<String, serde_json::Value>,
) -> impl Iterator<Item = (&'static str, &serde_json::Value)> {
PASSTHROUGH_EXTRA_FIELDS
.iter()
.filter_map(|key| extra_fields.get(*key).map(|value| (*key, value)))
}

static IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS: LazyLock<bool> =
LazyLock::new(|| env_is_truthy(DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS));

Expand Down Expand Up @@ -1305,6 +1316,24 @@ mod tests {
validate_no_unsupported_fields_with_ignore(&unknown_fields(), true).unwrap();
}

#[test]
fn passthrough_extra_fields_skips_ignored_unknown_fields() {
let extra_fields = HashMap::from([
("experimental_field".to_string(), json!("value")),
("stop_token_ids".to_string(), json!([7])),
("allowed_token_ids".to_string(), json!([10, 11])),
]);

let fields: Vec<_> = passthrough_extra_fields(&extra_fields).collect();
assert_eq!(
fields,
vec![
("stop_token_ids", &json!([7])),
("allowed_token_ids", &json!([10, 11])),
]
);
}

#[test]
fn validate_no_unsupported_fields_still_validates_passthrough_fields_when_ignoring_unknowns() {
let unsupported_fields = HashMap::from([
Expand Down
Loading