Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions lib/llm/src/http/service/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3618,10 +3618,16 @@ async fn audio_speech(

let mut response_collector = state.metrics_clone().create_response_collector(&model);

let stream = engine
.generate(request)
.await
.map_err(|e| ErrorMessage::from_anyhow(e, "Failed to generate audio"))?;
let stream = engine.generate(request).await.map_err(|e| {
if super::metrics::request_was_rejected(e.as_ref()) {
state
.metrics_clone()
.inc_rejection(&model, super::metrics::Endpoint::Audios);
}
let err_response = ErrorMessage::from_anyhow(e, "Failed to generate audio");
inflight.mark_error(extract_error_type_from_response(&err_response));
err_response
})?;

let mut http_queue_guard = Some(http_queue_guard);
let stream = stream.inspect(move |response| {
Expand All @@ -3635,12 +3641,17 @@ async fn audio_speech(
let response = NvAudioSpeechResponse::from_annotated_stream(stream)
.await
.map_err(|e| {
tracing::error!("Failed to fold audio stream for {}: {:?}", request_id, e);
ErrorMessage::internal_server_error("Failed to fold audio stream")
let err_response =
ErrorMessage::from_anyhow(anyhow::Error::new(e), "Failed to fold audio stream");
inflight.mark_error(extract_error_type_from_response(&err_response));
err_response
})?;

// Check for failure before marking success
if response.status == "failed" {
// Without this the guard drops on its default and books this 400 as an
// internal error.
inflight.mark_error(ErrorType::Validation);
return Ok((axum::http::StatusCode::BAD_REQUEST, Json(response)).into_response());
}

Expand Down
3 changes: 2 additions & 1 deletion lib/llm/src/protocols/openai/audios/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use futures::Stream;

use crate::protocols::openai::stream_aggregator::{StreamAggregable, aggregate_stream};
use crate::types::Annotated;
use dynamo_runtime::error::DynamoError;

use super::NvAudioSpeechResponse;

Expand All @@ -22,7 +23,7 @@ impl NvAudioSpeechResponse {
/// Aggregates an annotated stream of audio responses into a final response.
pub async fn from_annotated_stream(
stream: impl Stream<Item = Annotated<NvAudioSpeechResponse>>,
) -> Result<NvAudioSpeechResponse, String> {
) -> Result<NvAudioSpeechResponse, DynamoError> {
aggregate_stream(stream).await
}
}
84 changes: 52 additions & 32 deletions lib/llm/src/protocols/openai/chat_completions/aggregator.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use futures::{Stream, StreamExt};
use futures::{Stream, StreamExt, TryStreamExt};
use std::collections::{BTreeMap, HashMap};

use dynamo_parsers::tool_calling::try_tool_call_parse_aggregate_finalize;
Expand All @@ -17,6 +17,7 @@ use crate::protocols::{

use dynamo_protocols::types::ChatCompletionMessageContent;
use dynamo_runtime::engine::DataStream;
use dynamo_runtime::error::DynamoError;

fn is_harmony_parser(parser: &str) -> bool {
parser == "harmony"
Expand All @@ -42,8 +43,6 @@ pub struct DeltaAggregator {
system_fingerprint: Option<String>,
/// Map of incremental response choices, keyed by index.
choices: HashMap<u32, DeltaChoice>,
/// Optional error message if an error occurs during aggregation.
error: Option<String>,
/// Optional service tier information for the response.
service_tier: Option<dynamo_protocols::types::ServiceTierResponse>,
/// Aggregated nvext field from stream responses
Expand Down Expand Up @@ -199,7 +198,6 @@ impl DeltaAggregator {
usage: None,
system_fingerprint: None,
choices: HashMap::new(),
error: None,
service_tier: None,
nvext: None,
}
Expand All @@ -213,25 +211,15 @@ impl DeltaAggregator {
///
/// # Returns
/// * `Ok(NvCreateChatCompletionResponse)` if aggregation is successful.
/// * `Err(String)` if an error occurs during processing.
/// * `Err(DynamoError)` if an error occurs during processing.
pub async fn apply(
stream: impl Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateChatCompletionResponse, String> {
) -> Result<NvCreateChatCompletionResponse, DynamoError> {
let mut aggregator = stream
.fold(DeltaAggregator::new(), |mut aggregator, delta| async move {
// Attempt to unwrap the delta, capturing any errors.
let delta = match delta.ok() {
Ok(delta) => delta,
Err(error) => {
aggregator.error = Some(error);
return aggregator;
}
};

if aggregator.error.is_none()
&& let Some(delta) = delta.data
{
.map(Annotated::into_data)
.try_fold(DeltaAggregator::new(), |mut aggregator, delta| async move {
if let Some(delta) = delta {
aggregator.id = delta.inner.id;
aggregator.model = delta.inner.model;
aggregator.created = delta.inner.created;
Expand Down Expand Up @@ -339,14 +327,9 @@ impl DeltaAggregator {
}
}
}
aggregator
Ok(aggregator)
})
.await;

// Return early if an error was encountered.
if let Some(error) = aggregator.error {
return Err(error);
}
.await?;

// #8640: finalize the per-index tool-call chunk accumulator into the
// choice's `tool_calls` vector. Chunks missing id or name across the
Expand Down Expand Up @@ -523,11 +506,11 @@ pub trait ChatCompletionAggregator {
///
/// # Returns
/// * `Ok(NvCreateChatCompletionResponse)` if aggregation succeeds.
/// * `Err(String)` if an error occurs.
/// * `Err(DynamoError)` if an error occurs.
async fn from_annotated_stream(
stream: impl Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateChatCompletionResponse, String>;
) -> Result<NvCreateChatCompletionResponse, DynamoError>;

/// Converts an SSE stream into a [`NvCreateChatCompletionResponse`].
///
Expand All @@ -536,25 +519,25 @@ pub trait ChatCompletionAggregator {
///
/// # Returns
/// * `Ok(NvCreateChatCompletionResponse)` if aggregation succeeds.
/// * `Err(String)` if an error occurs.
/// * `Err(DynamoError)` if an error occurs.
async fn from_sse_stream(
stream: DataStream<Result<Message, SseCodecError>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateChatCompletionResponse, String>;
) -> Result<NvCreateChatCompletionResponse, DynamoError>;
}

impl ChatCompletionAggregator for NvCreateChatCompletionResponse {
async fn from_annotated_stream(
stream: impl Stream<Item = Annotated<NvCreateChatCompletionStreamResponse>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateChatCompletionResponse, String> {
) -> Result<NvCreateChatCompletionResponse, DynamoError> {
DeltaAggregator::apply(stream, parsing_options).await
}

async fn from_sse_stream(
stream: DataStream<Result<Message, SseCodecError>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateChatCompletionResponse, String> {
) -> Result<NvCreateChatCompletionResponse, DynamoError> {
let stream = convert_sse_stream::<NvCreateChatCompletionStreamResponse>(stream);
NvCreateChatCompletionResponse::from_annotated_stream(stream, parsing_options).await
}
Expand Down Expand Up @@ -1826,4 +1809,41 @@ mod tests {
);
assert!(json.get("reasoning_content").is_some());
}

#[tokio::test]
async fn preserves_typed_error_after_partial_output() {
use dynamo_runtime::error::{BackendError, ErrorType};

let partial = create_test_delta(
0,
"partial",
Some(dynamo_protocols::types::Role::Assistant),
None,
None,
None,
);
let error = DynamoError::builder()
.error_type(ErrorType::Backend(BackendError::InvalidArgument))
.message("invalid sampling parameter")
.build();
let stream = stream::iter(vec![
partial,
Annotated {
data: None,
id: None,
event: Some("error".to_string()),
comment: None,
error: Some(error),
},
]);

let error = DeltaAggregator::apply(stream, ParsingOptions::default())
.await
.unwrap_err();
assert_eq!(
error.error_type(),
ErrorType::Backend(BackendError::InvalidArgument)
);
assert_eq!(error.message(), "invalid sampling parameter");
}
}
67 changes: 39 additions & 28 deletions lib/llm/src/protocols/openai/completions/aggregator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@

use std::collections::HashMap;

use anyhow::Result;
use futures::{Stream, StreamExt};
use dynamo_runtime::error::DynamoError;
use futures::{Stream, StreamExt, TryStreamExt};

use super::NvCreateCompletionResponse;
use crate::protocols::{
Expand All @@ -23,7 +23,6 @@ pub struct DeltaAggregator {
usage: Option<dynamo_protocols::types::CompletionUsage>,
system_fingerprint: Option<String>,
choices: HashMap<u32, DeltaChoice>,
error: Option<String>,
nvext: Option<serde_json::Value>,
}

Expand All @@ -49,7 +48,6 @@ impl DeltaAggregator {
usage: None,
system_fingerprint: None,
choices: HashMap::new(),
error: None,
nvext: None,
}
}
Expand All @@ -58,21 +56,12 @@ impl DeltaAggregator {
pub async fn apply(
stream: impl Stream<Item = Annotated<NvCreateCompletionResponse>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateCompletionResponse> {
) -> Result<NvCreateCompletionResponse, DynamoError> {
tracing::debug!("Tool Call Parser: {:?}", parsing_options.tool_call_parser); // TODO: remove this once completion has tool call support
let aggregator = stream
.fold(DeltaAggregator::new(), |mut aggregator, delta| async move {
let delta = match delta.ok() {
Ok(delta) => delta,
Err(error) => {
aggregator.error = Some(error);
return aggregator;
}
};

if aggregator.error.is_none()
&& let Some(delta) = delta.data
{
.map(Annotated::into_data)
.try_fold(DeltaAggregator::new(), |mut aggregator, delta| async move {
if let Some(delta) = delta {
// TODO(#14) - Aggregate Annotation

// these are cheap to move so we do it every time since we are consuming the delta
Expand Down Expand Up @@ -137,16 +126,9 @@ impl DeltaAggregator {
}
}
}
aggregator
Ok(aggregator)
})
.await;

// If we have an error, return it
let aggregator = if let Some(error) = aggregator.error {
return Err(anyhow::anyhow!(error));
} else {
aggregator
};
.await?;

// extra the aggregated deltas and sort by index
let mut choices: Vec<_> = aggregator
Expand Down Expand Up @@ -193,15 +175,15 @@ impl NvCreateCompletionResponse {
pub async fn from_sse_stream(
stream: DataStream<Result<Message, SseCodecError>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateCompletionResponse> {
) -> Result<NvCreateCompletionResponse, DynamoError> {
let stream = convert_sse_stream::<NvCreateCompletionResponse>(stream);
NvCreateCompletionResponse::from_annotated_stream(stream, parsing_options).await
}

pub async fn from_annotated_stream(
stream: impl Stream<Item = Annotated<NvCreateCompletionResponse>>,
parsing_options: ParsingOptions,
) -> Result<NvCreateCompletionResponse> {
) -> Result<NvCreateCompletionResponse, DynamoError> {
DeltaAggregator::apply(stream, parsing_options).await
}
}
Expand Down Expand Up @@ -463,4 +445,33 @@ mod tests {
Some(dynamo_protocols::types::CompletionFinishReason::Stop)
);
}

#[tokio::test]
async fn preserves_typed_error_after_partial_output() {
use dynamo_runtime::error::{BackendError, ErrorType};

let error = DynamoError::builder()
.error_type(ErrorType::Backend(BackendError::InvalidArgument))
.message("unsupported logprobs")
.build();
let stream = stream::iter(vec![
create_test_delta(0, "partial", None, None),
Annotated {
data: None,
id: None,
event: Some("error".to_string()),
comment: None,
error: Some(error),
},
]);

let error = DeltaAggregator::apply(stream, ParsingOptions::default())
.await
.unwrap_err();
assert_eq!(
error.error_type(),
ErrorType::Backend(BackendError::InvalidArgument)
);
assert_eq!(error.message(), "unsupported logprobs");
}
}
Loading
Loading