Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/fern/pages/use-cases/agents/agent-tracing.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ Default request traces do not capture prompt, response, or tool-call argument bo

Requests with recognized session identity can include opaque strings in `agent_context.agent_headers`. Trace consumers decide how to interpret them and must tolerate unknown names, values, and embedded JSON. Captured headers do not change request placement or evict cached blocks. See [Captured agent headers](session-ids.mdx#agent-headers) for capture rules and limits. Older frontends cannot supply the map during a rolling upgrade.

For chat streams, Dynamo records finish metadata after parser and jail rewrites. Completion streams record the final OpenAI-compatible completion finish reason.
For chat streams, Dynamo records finish metadata after parser and jail rewrites. Completion streams record the final OpenAI-compatible completion finish reason. On `/v1/responses`, each `tool_calls` entry records the tool name the client receives, and a `namespace` field for a tool in a namespace.

<Warning>
Request tracing covers eligible chat completions, completions, Responses, and Messages requests that pass through the Rust preprocessor. It skips unsupported replay shapes, including `n > 1`, `best_of > 1`, `prompt_embeds`, multimodal inputs, and requests without a tracker or usable KV cache block size. Sinks use best-effort delivery and can drop records when they lag, so check warnings and validate row counts before treating a capture as complete.
Expand Down
10 changes: 10 additions & 0 deletions lib/llm/src/http/service/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4597,6 +4597,16 @@ async fn responses(
if response_params.max_output_tokens.is_none() {
request.insert(PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY, true);
}
// Clients get public tool names and namespaces, not backend aliases;
// request traces record them. Skip the map clone when traces are off.
if crate::request_trace::policy().emit_request_end_records()
&& let Some(tool_names) = response_params.tool_names.clone()
{
request.insert(
crate::request_trace::CLIENT_TOOL_NAMES_CONTEXT_KEY,
crate::request_trace::ClientToolNames(std::sync::Arc::new(tool_names)),
);
}

tracing::trace!("Getting chat completions engine for model: {}", model);

Expand Down
2 changes: 1 addition & 1 deletion lib/llm/src/protocols/openai/responses/tool_names.rs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ impl ToolNameMap {
}

/// Restore the original namespace and name, treating unknown aliases as bare names.
pub(super) fn decode<'a>(&'a self, alias: &'a str) -> (Option<&'a str>, &'a str) {
pub(crate) fn decode<'a>(&'a self, alias: &'a str) -> (Option<&'a str>, &'a str) {
self.by_alias
.get(alias)
.map(|(namespace, name)| (namespace.as_deref(), name.as_str()))
Expand Down
96 changes: 96 additions & 0 deletions lib/llm/src/request_trace/agent_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ impl FinishReasonMetadataState {
tool_call_index,
id: pending.id.clone(),
name: pending.name.clone(),
namespace: None,
});
}
}
Expand Down Expand Up @@ -241,6 +242,8 @@ pub(crate) struct AgentContextTraceState {
pub finish_reason_metadata: SharedFinishReasonMetadata,
/// Set when the client receives other tool-call IDs than the backend's.
pub client_tool_call_ids: Option<super::ClientToolCallIds>,
/// Set when the client receives other tool names than the backend's.
pub client_tool_names: Option<super::ClientToolNames>,
}

pub(crate) fn build_agent_context_trace_state(
Expand All @@ -267,6 +270,10 @@ pub(crate) fn build_agent_context_trace_state(
.get::<super::ClientToolCallIds>(super::CLIENT_TOOL_CALL_IDS_CONTEXT_KEY)
.ok()
.map(|ids| *ids),
client_tool_names: context
.get::<super::ClientToolNames>(super::CLIENT_TOOL_NAMES_CONTEXT_KEY)
.ok()
.map(|names| names.as_ref().clone()),
})
}

Expand Down Expand Up @@ -371,6 +378,7 @@ pub(crate) fn request_metrics_from_agent_state(
x_request_id,
finish_reason_metadata,
client_tool_call_ids,
client_tool_names,
} = trace_state;

if request_tracker.is_none() {
Expand Down Expand Up @@ -409,6 +417,20 @@ pub(crate) fn request_metrics_from_agent_state(
*id = derive(&chat_id, id, occurrence);
}
}
if let (Some(super::ClientToolNames(names)), Some(metadata)) =
(client_tool_names, metrics.finish_reason_metadata.as_mut())
{
// The same decode as the Responses stream converter, so the trace
// records the identity the client received.
for call in &mut metadata.tool_calls {
let Some(name) = call.name.as_deref() else {
continue;
};
let (namespace, leaf) = names.decode(name);
call.namespace = namespace.map(str::to_owned);
call.name = Some(leaf.to_owned());
}
}
(agent_context, metrics)
}

Expand Down Expand Up @@ -602,6 +624,7 @@ mod tests {
format!("client-{chat_id}-{backend_id}-{occurrence}")
},
)),
client_tool_names: None,
};

let (_, request) = request_metrics_from_agent_state(trace_state, "r1".to_string());
Expand All @@ -621,6 +644,65 @@ mod tests {
]
);
}
#[test]
fn request_end_records_public_responses_tool_names() {
let tools: Vec<dynamo_protocols::types::responses::Tool> =
serde_json::from_value(serde_json::json!([
{"type":"namespace", "name":"crm", "description":"CRM",
"tools":[{"type":"function", "name":"lookup"}]},
{"type":"namespace", "name":"billing", "description":"Billing",
"tools":[{"type":"function", "name":"lookup"}]},
{"type":"function", "name":"shell"}
]))
.unwrap();
let names = crate::protocols::openai::responses::ToolNameMap::new(&tools, None);
let finish_reason_metadata = SharedFinishReasonMetadata::default();
{
let mut metadata = finish_reason_metadata.lock();
// A collision alias, a native qualified name, and a plain name.
metadata.record_tool_call_chunk(0, 0, Some("call_1"), Some("crm__lookup"));
metadata.record_tool_call_chunk(0, 1, Some("call_2"), Some("billing.lookup"));
metadata.record_tool_call_chunk(0, 2, Some("call_3"), Some("shell"));
}
let trace_state = AgentContextTraceState {
agent_context: AgentContext {
session_id: "client-names".to_string(),
parent_session_id: None,
session_final: None,
agent_headers: Default::default(),
input_trigger: None,
},
request_model: "test-model".to_string(),
request_tracker: None,
x_request_id: None,
finish_reason_metadata,
client_tool_call_ids: None,
client_tool_names: Some(super::super::ClientToolNames(std::sync::Arc::new(names))),
};

let (_, request) = request_metrics_from_agent_state(trace_state, "r1".to_string());

let metadata = request.finish_reason_metadata.expect("finish metadata");
let identities: Vec<_> = metadata
.tool_calls
.iter()
.map(|call| (call.name.as_deref(), call.namespace.as_deref()))
.collect();
assert_eq!(
identities,
vec![
(Some("lookup"), Some("crm")),
(Some("lookup"), Some("billing")),
(Some("shell"), None),
]
);
let value = serde_json::to_value(&metadata.tool_calls).unwrap();
assert_eq!(value[0]["namespace"], "crm");
assert!(value[2].get("namespace").is_none());
// IDs are not rewritten without `client_tool_call_ids`.
assert_eq!(metadata.tool_calls[0].id.as_deref(), Some("call_1"));
}

#[test]
fn test_request_metrics_from_tracker() {
let tracker = RequestTracker::new();
Expand Down Expand Up @@ -719,6 +801,7 @@ mod tests {
x_request_id: Some("llm-call-1".to_string()),
finish_reason_metadata,
client_tool_call_ids: None,
client_tool_names: None,
};

let responses = vec![
Expand Down Expand Up @@ -812,6 +895,18 @@ mod tests {
assert_eq!(metadata.tool_calls[0].tool_call_index, 0);
assert_eq!(metadata.tool_calls[0].id.as_deref(), Some("call-1"));
assert_eq!(metadata.tool_calls[0].name.as_deref(), Some("web_search"));
// Without a Responses name map, chat rows keep the raw name and gain no key.
assert_eq!(metadata.tool_calls[0].namespace, None);
let tool_call = serde_json::to_value(&metadata.tool_calls[0]).unwrap();
assert_eq!(
tool_call,
serde_json::json!({
"choice_index": 0,
"tool_call_index": 0,
"id": "call-1",
"name": "web_search"
})
);
assert_eq!(metadata.choices.len(), 1);
assert_eq!(metadata.choices[0].choice_index, 0);
assert_eq!(
Expand Down Expand Up @@ -847,6 +942,7 @@ mod tests {
x_request_id: Some("completion-call-1".to_string()),
finish_reason_metadata,
client_tool_call_ids: None,
client_tool_names: None,
};

let responses = vec![Annotated::from_data(NvCreateCompletionResponse {
Expand Down
28 changes: 28 additions & 0 deletions lib/llm/src/request_trace/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ mod tests {
x_request_id: None,
finish_reason_metadata: Default::default(),
client_tool_call_ids: None,
client_tool_names: None,
});
if already_finished {
tracker.record_finish();
Expand Down Expand Up @@ -588,6 +589,33 @@ mod tests {
);
}

#[test]
fn agent_context_reads_client_tool_names_from_context() {
let mut request = preprocessed_request(SamplingOptions::default());
request.agent_context = Some(AgentContext {
session_id: "root".to_string(),
parent_session_id: None,
session_final: None,
agent_headers: Default::default(),
input_trigger: None,
});
let tracker = Some(Arc::new(RequestTracker::new()));

let plain = Context::new(());
let state =
build_request_end_trace_state_for_policy(&request, &tracker, &plain, 2, true).unwrap();
assert!(state.agent.as_ref().unwrap().client_tool_names.is_none());

let mut context = Context::new(());
context.insert(
crate::request_trace::CLIENT_TOOL_NAMES_CONTEXT_KEY,
crate::request_trace::ClientToolNames(Arc::new(Default::default())),
);
let state = build_request_end_trace_state_for_policy(&request, &tracker, &context, 2, true)
.unwrap();
assert!(state.agent.as_ref().unwrap().client_tool_names.is_some());
}

#[test]
fn agent_context_does_not_bypass_request_trace_eligibility() {
let mut request = preprocessed_request(SamplingOptions {
Expand Down
13 changes: 13 additions & 0 deletions lib/llm/src/request_trace/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ pub mod sink;
mod tool_relay;
pub mod types;

use std::sync::Arc;

use tokio_util::sync::CancellationToken;

use dynamo_runtime::DistributedRuntime;

use crate::local_model::LocalModel;
use crate::protocols::openai::responses::ToolNameMap;
use crate::telemetry::bus::TelemetryBus;

pub use agent_context::SharedFinishReasonMetadata;
Expand Down Expand Up @@ -65,6 +68,16 @@ pub const CLIENT_TOOL_CALL_IDS_CONTEXT_KEY: &str = "request_trace.client_tool_ca
#[derive(Clone, Copy)]
pub struct ClientToolCallIds(pub fn(&str, &str, u32) -> String);

/// Context key for an endpoint that gives clients other tool names than the
/// backend's; its value is the endpoint's [`ClientToolNames`].
pub const CLIENT_TOOL_NAMES_CONTEXT_KEY: &str = "request_trace.client_tool_names";

/// The request's Responses tool-name map, so `request_end` records each tool
/// call's public leaf name and namespace as the client receives them, not the
/// backend's qualified name or collision alias.
#[derive(Clone)]
pub struct ClientToolNames(pub Arc<ToolNameMap>);

pub async fn init_from_env_with_shutdown(shutdown: CancellationToken) -> anyhow::Result<()> {
let policy = policy();
if !policy.enabled {
Expand Down
4 changes: 4 additions & 0 deletions lib/llm/src/request_trace/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,10 @@ pub struct ToolCallMetadata {
pub id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// Set only when the endpoint returns the tool's namespace separately from
/// `name`, as `/v1/responses` does for namespaced tools.
#[serde(skip_serializing_if = "Option::is_none")]
pub namespace: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
Expand Down
Loading