Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 65 additions & 91 deletions lib/bindings/python/rust/llm/py_convert.rs
Original file line number Diff line number Diff line change
@@ -1,79 +1,60 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Direct reads of plain Python data into Rust values, for the request fields that carry
//! prompt-sized lists of ints. pythonize converts each int through pyo3's abi3 128-bit
//! extraction and each list item through the generic sequence protocol; these read exact
//! builtin types straight through the C API instead.
//!
//! Each function returns `None` when the input holds anything outside the exact builtin
//! types it reads. The caller then converts with pythonize as before, so results and
//! errors for those inputs are unchanged. For every input accepted, the result equals
//! pythonize's.
//! Direct reads of exact builtin Python types, for request fields that carry prompt-sized
//! lists of ints, where pythonize pays tens of ns per int. Accepted inputs convert exactly
//! as pythonize would; anything else returns `None`, and the caller falls back to pythonize.

use pyo3::ffi;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyFloat, PyList, PyString, PyTuple};
use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTuple};
use serde_json::{Map, Number, Value};

/// The map key serde_json's `raw_value` feature reads as embedded raw JSON
/// (`serde_json::raw::TOKEN`, which is private).
const SERDE_JSON_RAW_VALUE_TOKEN: &str = "$serde_json::private::RawValue";

/// Nesting depth past which [`exact_json_value`] declines rather than recursing further.
const MAX_DEPTH: usize = 128;

/// An exact `list` of exact `int`s as `Vec<u32>`; `None` for anything else, including a
/// negative id or one past `u32::MAX`.
pub(crate) fn read_exact_u32_list(obj: &Bound<'_, PyAny>) -> Option<Vec<u32>> {
let list = obj.downcast_exact::<PyList>().ok()?;
let mut out = Vec::with_capacity(list.len());
// SAFETY: the GIL is held and the loop runs no Python code (only exact `int`s are
// converted, and `PyLong_AsUnsignedLong` does not call `__index__`), so the exact
// list cannot change, every index is in range, and the borrowed items stay valid.
// SAFETY: `PyLong_AsUnsignedLong` on an exact `int` calls no `__index__`, so the read
// runs no Python code.
unsafe {
for index in 0..list.len() {
let item = ffi::PyList_GetItem(list.as_ptr(), index as ffi::Py_ssize_t);
if ffi::PyLong_CheckExact(item) == 0 {
return None;
}
read_list(list, |item| {
let int = item.downcast_exact::<PyInt>().ok()?;
// A negative int sets OverflowError and returns `c_ulong::MAX`; both it and
// values past `u32::MAX` fail `try_from`.
let Ok(id) = u32::try_from(ffi::PyLong_AsUnsignedLong(item)) else {
let id = u32::try_from(ffi::PyLong_AsUnsignedLong(int.as_ptr()));
if id.is_err() {
ffi::PyErr_Clear();
return None;
};
out.push(id);
}
}
id.ok()
})
}
Some(out)
}

/// Plain JSON-shaped Python data (exact `dict` with `str` keys, `list`, `tuple`, `str`,
/// `int`, `float`, `bool`, `None`) as a `serde_json::Value`; `None` for anything else.
/// Exact `dict` (with `str` keys), `list`, `tuple`, `str`, `int` (in `i64` or `u64` range),
/// `float`, `bool` or `None` as a `serde_json::Value`; `None` for anything else.
pub(crate) fn exact_json_value(obj: &Bound<'_, PyAny>) -> Option<Value> {
Comment thread
jthomson04 marked this conversation as resolved.
Resolved
json_value(obj, 0)
}

fn json_value(obj: &Bound<'_, PyAny>, depth: usize) -> Option<Value> {
if depth > MAX_DEPTH {
return None;
if let Ok(int) = obj.downcast_exact::<PyInt>() {
return exact_int(int);
}
let ptr = obj.as_ptr();
// SAFETY: `ptr` is a live object; these are type checks and reads only.
unsafe {
if ffi::PyLong_CheckExact(ptr) != 0 {
return exact_int(ptr);
}
if ffi::PyList_CheckExact(ptr) != 0 {
return sequence(obj, ptr, depth, ffi::PyList_GetItem);
}
if let Ok(list) = obj.downcast_exact::<PyList>() {
// SAFETY: this function reads only exact builtin types, which runs no Python code.
let items = unsafe {
read_list(list, |item| match item.downcast_exact::<PyInt>() {
// Ints, by far the commonest item, skip the recursive call.
Ok(int) => exact_int(int),
Err(_) => exact_json_value(item),
})
};
return items.map(Value::Array);
}
if obj.is_none() {
return Some(Value::Null);
}
// `bool` cannot be subclassed, so this is exact; pythonize reads it before `int`.
if let Ok(flag) = obj.downcast_exact::<pyo3::types::PyBool>() {
if let Ok(flag) = obj.downcast_exact::<PyBool>() {
return Some(Value::Bool(flag.is_true()));
}
if let Ok(text) = obj.downcast_exact::<PyString>() {
Expand All @@ -83,30 +64,29 @@ fn json_value(obj: &Bound<'_, PyAny>, depth: usize) -> Option<Value> {
// NaN and the infinities become `Null`, as serde_json's own `Value` visitor does.
return Some(Number::from_f64(number.value()).map_or(Value::Null, Value::Number));
}
if obj.downcast_exact::<PyTuple>().is_ok() {
// SAFETY: as above.
return unsafe { sequence(obj, obj.as_ptr(), depth, ffi::PyTuple_GetItem) };
if let Ok(tuple) = obj.downcast_exact::<PyTuple>() {
return tuple
.iter_borrowed()
.map(|item| exact_json_value(&item))
.collect::<Option<_>>()
.map(Value::Array);
}
if let Ok(dict) = obj.downcast_exact::<PyDict>() {
let mut map = Map::with_capacity(dict.len());
for (key, value) in dict.iter() {
let key = key.downcast_exact::<PyString>().ok()?.to_str().ok()?;
if key == SERDE_JSON_RAW_VALUE_TOKEN {
return None;
}
map.insert(key.to_owned(), json_value(&value, depth + 1)?);
let dict = obj.downcast_exact::<PyDict>().ok()?;
let mut map = Map::with_capacity(dict.len());
for (key, value) in dict.iter() {
let key = key.downcast_exact::<PyString>().ok()?.to_str().ok()?;
if key == SERDE_JSON_RAW_VALUE_TOKEN {
return None;
}
return Some(Value::Object(map));
map.insert(key.to_owned(), exact_json_value(&value)?);
}
None
Some(Value::Object(map))
}
Comment thread
jthomson04 marked this conversation as resolved.
Resolved

/// An exact `int` in `i64` or `u64` range, as pythonize gives it to serde_json: the same
/// `Number`, whichever of the two ranges it falls in. Larger ints decline.
///
/// # Safety
/// `ptr` must be a live exact `int`, with the GIL held.
unsafe fn exact_int(ptr: *mut ffi::PyObject) -> Option<Value> {
/// An `int` in `i64` or `u64` range as the same `Number` pythonize gives; larger ints decline.
fn exact_int(int: &Bound<'_, PyInt>) -> Option<Value> {
let ptr = int.as_ptr();
// SAFETY: `ptr` is a live `int` and the GIL is held; these calls run no Python code.
unsafe {
let signed = ffi::PyLong_AsLongLong(ptr);
if signed != -1 || ffi::PyErr_Occurred().is_null() {
Expand All @@ -122,32 +102,26 @@ unsafe fn exact_int(ptr: *mut ffi::PyObject) -> Option<Value> {
}
}

/// An exact `list` or `tuple` as an array, reading items by index.
/// Each item of `list` through `read`, borrowed without touching its refcount; `None` if
/// `read` declines any item.
///
/// # Safety
/// `ptr` must be `obj`'s pointer and `get_item` the matching borrowed-item getter, with the
/// GIL held.
unsafe fn sequence(
obj: &Bound<'_, PyAny>,
ptr: *mut ffi::PyObject,
depth: usize,
get_item: unsafe extern "C" fn(*mut ffi::PyObject, ffi::Py_ssize_t) -> *mut ffi::PyObject,
) -> Option<Value> {
let len = obj.len().ok()?;
let mut items = Vec::with_capacity(len);
// SAFETY: the conversion runs no Python code, so the exact list or tuple cannot change,
// every index is in range, and the borrowed items stay valid while they are read.
unsafe {
for index in 0..len {
let item = get_item(ptr, index as ffi::Py_ssize_t);
// Ints, by far the commonest item, skip building a `Bound`.
let value = if ffi::PyLong_CheckExact(item) != 0 {
exact_int(item)?
} else {
json_value(&Bound::from_borrowed_ptr(obj.py(), item), depth + 1)?
};
items.push(value);
}
/// `read` must run no Python code, so that nothing can change the list while it is read.
unsafe fn read_list<T>(
list: &Bound<'_, PyList>,
read: impl Fn(&Bound<'_, PyAny>) -> Option<T>,
) -> Option<Vec<T>> {
let mut out = Vec::with_capacity(list.len());
for index in 0..list.len() {
// SAFETY: the GIL is held and, per the contract, the list cannot change, so every
// index is in range and the borrowed item stays valid while `read` uses it.
let item = unsafe {
Borrowed::from_ptr(
list.py(),
ffi::PyList_GetItem(list.as_ptr(), index as ffi::Py_ssize_t),
)
};
out.push(read(&item)?);
}
Some(Value::Array(items))
Some(out)
}
38 changes: 13 additions & 25 deletions lib/bindings/python/rust/llm/routed_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,43 +32,29 @@ impl RoutedEngine {
}
}

/// The `PreprocessedRequest` a Python engine passes to [`RoutedEngine::generate`].
///
/// `token_ids` and `extra_args` can each hold a prompt-sized list of ints (a chat engine's
/// capture plan, a multimodal prompt's expanded ids), and pythonize costs tens of ns per
/// int on the event loop's thread. [`fast_preprocessed_request`] reads those two fields
/// directly; any request it declines takes the full pythonize conversion, so results and
/// errors for it are unchanged.
fn preprocessed_request_from_py(obj: &Bound<'_, PyAny>) -> PyResult<PreprocessedRequest> {
if let Some(request) = fast_preprocessed_request(obj) {
return Ok(request);
}
depythonize(obj).map_err(to_pyerr)
}

/// An exact `dict` whose `token_ids` is an exact list of ints and whose `extra_args` is
/// absent, `None` or plain JSON-shaped data, with those two fields read directly and the
/// rest by pythonize. `None` for anything else, or if the rest fails to convert.
/// The request with `token_ids` and `extra_args` read by [`py_convert`] and the rest by
/// pythonize; `None` unless it is an exact `dict` whose two fields both read.
fn fast_preprocessed_request(obj: &Bound<'_, PyAny>) -> Option<PreprocessedRequest> {
let py = obj.py();
let token_ids_key = pyo3::intern!(py, "token_ids");
let extra_args_key = pyo3::intern!(py, "extra_args");
let dict = obj.downcast_exact::<PyDict>().ok()?;
let token_ids = py_convert::read_exact_u32_list(&dict.get_item("token_ids").ok()??)?;
let extra_args = match dict.get_item("extra_args").ok()? {
let token_ids = py_convert::read_exact_u32_list(&dict.get_item(token_ids_key).ok()??)?;
let extra_args = match dict.get_item(extra_args_key).ok()? {
Some(value) if !value.is_none() => Some(py_convert::exact_json_value(&value)?),
_ => None,
};
// `token_ids` is required, so the copy keeps an empty one for pythonize to read.
let rest = dict.copy().ok()?;
rest.set_item("token_ids", PyList::empty(obj.py())).ok()?;
rest.set_item("extra_args", obj.py().None()).ok()?;
rest.set_item(token_ids_key, PyList::empty(py)).ok()?;
rest.set_item(extra_args_key, py.None()).ok()?;
let mut request: PreprocessedRequest = depythonize(rest.as_any()).ok()?;
request.token_ids = Arc::new(token_ids);
request.extra_args = extra_args;
Some(request)
}

/// Test hook: the request [`RoutedEngine::generate`] builds from `preprocessed`, pythonized
/// back. With `fast_path`, only the direct read (`None` when it declines); without, full
/// pythonize. Tests compare the two.
/// Test hook: the fast path's request (`None` if it declines) or pythonize's, pythonized back.
#[pyfunction]
#[pyo3(signature = (preprocessed, fast_path))]
pub fn _convert_preprocessed_request(
Expand Down Expand Up @@ -105,7 +91,9 @@ impl RoutedEngine {
preprocessed: PyObject,
context: Option<crate::context::Context>,
) -> PyResult<Bound<'p, PyAny>> {
let request = preprocessed_request_from_py(preprocessed.bind(py))?;
let preprocessed = preprocessed.bind(py);
let request = fast_preprocessed_request(preprocessed)
.map_or_else(|| depythonize(preprocessed).map_err(to_pyerr), Ok)?;
let request_context = if let Some(parent_context) = context.as_ref() {
let parent_metadata = parent_context.metadata_snapshot();
let parent_controller = parent_context.inner();
Expand Down
6 changes: 1 addition & 5 deletions lib/bindings/python/src/dynamo/_core.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -2096,11 +2096,7 @@ def _normalize_mocker_config(config: Mapping[str, Any] | str) -> Dict[str, Any]:
def _convert_preprocessed_request(
preprocessed: Dict[str, Any], fast_path: bool
) -> Optional[Dict[str, Any]]:
"""Test hook: the request RoutedEngine.generate builds, pythonized back.

With fast_path, only the direct read of token_ids and extra_args (None when it
declines the input); without, full pythonize.
"""
"""Test hook: the fast path's request (None if it declines) or pythonize's, pythonized back."""
...


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,13 @@ class TokenList(list):
]

DECLINED = [
{"model": "m", "token_ids": (1, 2)},
{"model": "m", "token_ids": b"\x01\x00\x00\x00"},
{"model": "m", "token_ids": TokenList([1, 2])},
request(token_ids=[1, -1]),
request(token_ids=[1, 2**32]),
request(token_ids=[True, 2]),
request(token_ids=[1, -1]),
request(extra_args={"s": {1, 2}}),
request(extra_args={"b": b"x"}),
request(extra_args={1: "x"}),
request(extra_args={"big": 2**64}),
request(extra_args={"$serde_json::private::RawValue": "[1]"}),
]
Comment thread
jthomson04 marked this conversation as resolved.
Resolved


Expand Down
Loading