Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/codeowners/areas.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ areas:
- lib/sidecar/dynamo-sidecar
- lib/sidecar/README.md
- lib/sidecar/common/
- lib/sidecar/testkit/
- lib/bindings/
- lib/bindings/c/
- lib/bindings/python/
Expand Down
1 change: 1 addition & 0 deletions CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@
/lib/bindings/python/ @ai-dynamo/dynamo-runtime-codeowners
/lib/llm/src/kernels/ @ai-dynamo/dynamo-runtime-codeowners
/lib/runtime/benches/ @ai-dynamo/dynamo-runtime-codeowners
/lib/sidecar/testkit/ @ai-dynamo/dynamo-runtime-codeowners
/lib/runtime/examples/ @ai-dynamo/dynamo-runtime-codeowners
/lib/sidecar/README.md @ai-dynamo/dynamo-runtime-codeowners
/tests/serve/fixtures/ @ai-dynamo/dynamo-runtime-codeowners
Expand Down
3 changes: 3 additions & 0 deletions lib/sidecar/common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,6 @@ tonic = { workspace = true }
tonic-v14 = { workspace = true, optional = true }
tracing = { workspace = true }
url = { workspace = true }

[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
26 changes: 26 additions & 0 deletions lib/sidecar/common/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,21 @@ mod tests {
fn maps_transport_statuses_to_backend_errors() {
for (code, expected) in [
(tonic::Code::InvalidArgument, BackendError::InvalidArgument),
(tonic::Code::NotFound, BackendError::InvalidArgument),
(tonic::Code::OutOfRange, BackendError::InvalidArgument),
(
tonic::Code::FailedPrecondition,
BackendError::InvalidArgument,
),
(tonic::Code::AlreadyExists, BackendError::InvalidArgument),
(tonic::Code::Unknown, BackendError::Unknown),
(tonic::Code::Unimplemented, BackendError::Unknown),
(tonic::Code::ResourceExhausted, BackendError::Unknown),
(tonic::Code::PermissionDenied, BackendError::Unknown),
(tonic::Code::Unauthenticated, BackendError::Unknown),
(tonic::Code::Aborted, BackendError::Unknown),
(tonic::Code::DataLoss, BackendError::Unknown),
(tonic::Code::Ok, BackendError::Unknown),
(tonic::Code::Unavailable, BackendError::CannotConnect),
(tonic::Code::Cancelled, BackendError::Cancelled),
(
Expand All @@ -128,6 +143,17 @@ mod tests {
] {
let error = status_to_dynamo("Test", tonic::Status::new(code, "failure"));
assert_eq!(error.error_type(), ErrorType::Backend(expected));
assert!(error.to_string().contains("Test: failure"));
assert!(error.to_string().contains(&format!("{code:?}")));
#[cfg(feature = "tonic-v14")]
{
let v14 = crate::error::status_to_dynamo_v14(
"Test",
tonic_v14::Status::new(tonic_v14::Code::from_i32(code as i32), "failure"),
);
assert_eq!(v14.error_type(), error.error_type());
assert_eq!(v14.to_string(), error.to_string());
}
}
}
}
4 changes: 4 additions & 0 deletions lib/sidecar/common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,7 @@ pub use error::{
protocol_error, status_to_dynamo,
};
pub use transport::{DEFAULT_MAX_GRPC_MESSAGE_SIZE, GrpcChannelPool, format_error_chain};

#[cfg(test)]
#[path = "transport/tests.rs"]
mod transport_tests;
107 changes: 67 additions & 40 deletions lib/sidecar/common/src/transport.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,40 +39,17 @@ impl GrpcChannelPool {
let tonic_endpoint = Endpoint::from_shared(endpoint_label.clone()).map_err(|error| {
invalid_argument(format!("invalid {peer} endpoint after validation: {error}"))
})?;
let deadline = checked_instant_add(
Instant::now(),
transport.startup_deadline,
"gRPC startup deadline",
)?;
let first = connect_until_ready(
let channels = connect_pool_with(
peer,
tonic_endpoint.clone(),
endpoint_label.clone(),
1,
&endpoint_label,
transport,
deadline,
bootstrap,
|_, attempt_timeout| {
let endpoint = tonic_endpoint.clone().connect_timeout(attempt_timeout);
async move { endpoint.connect().await }
},
)
.await?;
let mut channels = vec![first];
let remaining = try_join_all((1..transport.connections.get()).map(|index| {
let endpoint = tonic_endpoint.clone();
let endpoint_label = endpoint_label.clone();
async move {
connect_until_ready(
peer,
endpoint,
endpoint_label,
index + 1,
transport,
deadline,
bootstrap,
)
.await
}
}))
.await?;
channels.extend(remaining);
Ok(Self {
channels,
next: AtomicUsize::new(0),
Expand All @@ -93,15 +70,63 @@ impl GrpcChannelPool {
}
}

async fn connect_until_ready(
pub(crate) async fn connect_pool_with<C, F, E>(
peer: &str,
endpoint: &str,
transport: GrpcTransportConfig,
bootstrap: bool,
connect: impl Fn(usize, Duration) -> F,
) -> Result<Vec<C>, DynamoError>
where
F: std::future::Future<Output = Result<C, E>>,
E: std::error::Error + 'static,
{
let deadline = checked_instant_add(
Instant::now(),
transport.startup_deadline,
"gRPC startup deadline",
)?;
let first = connect_until_ready(
peer,
endpoint,
1,
transport,
deadline,
bootstrap,
|timeout| connect(1, timeout),
)
.await?;
let mut channels = vec![first];
let remaining = try_join_all((2..=transport.connections.get()).map(|slot| {
let connect = &connect;
connect_until_ready(
peer,
endpoint,
slot,
transport,
deadline,
bootstrap,
move |timeout| connect(slot, timeout),
)
}))
.await?;
channels.extend(remaining);
Ok(channels)
}

async fn connect_until_ready<C, F, E>(
peer: &str,
endpoint: Endpoint,
endpoint_label: String,
endpoint_label: &str,
pool_slot: usize,
transport: GrpcTransportConfig,
deadline: Instant,
bootstrap: bool,
) -> Result<Channel, DynamoError> {
connect: impl Fn(Duration) -> F,
) -> Result<C, DynamoError>
where
F: std::future::Future<Output = Result<C, E>>,
E: std::error::Error + 'static,
{
let started = Instant::now();
let mut attempt = 0_u64;
let mut last_error = None;
Expand All @@ -113,7 +138,7 @@ async fn connect_until_ready(
if remaining.is_zero() {
return Err(startup_timeout(
peer,
&endpoint_label,
endpoint_label,
pool_slot,
attempt,
started.elapsed(),
Expand All @@ -123,10 +148,12 @@ async fn connect_until_ready(
}

attempt += 1;
let attempt_endpoint = endpoint
.clone()
.connect_timeout(transport.connect_attempt_timeout.min(remaining));
match timeout_at(deadline, attempt_endpoint.connect()).await {
match timeout_at(
deadline,
connect(transport.connect_attempt_timeout.min(remaining)),
)
.await
{
Ok(Ok(channel)) => return Ok(channel),
Ok(Err(error)) => {
let detailed_error = format_error_chain(&error);
Expand Down Expand Up @@ -173,7 +200,7 @@ async fn connect_until_ready(
Err(_) => {
return Err(startup_timeout(
peer,
&endpoint_label,
endpoint_label,
pool_slot,
attempt,
started.elapsed(),
Expand All @@ -186,7 +213,7 @@ async fn connect_until_ready(
if Instant::now() >= deadline {
return Err(startup_timeout(
peer,
&endpoint_label,
endpoint_label,
pool_slot,
attempt,
started.elapsed(),
Expand Down
93 changes: 93 additions & 0 deletions lib/sidecar/common/src/transport/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::cell::Cell;
use std::io;
use std::num::NonZeroUsize;
use std::time::Duration;

use crate::GrpcTransportConfig;
use crate::transport::connect_pool_with;
use dynamo_backend_common::{BackendError, ErrorType};
use tokio::time::{Instant, sleep};

fn config() -> GrpcTransportConfig {
GrpcTransportConfig {
connections: NonZeroUsize::new(3).unwrap(),
connect_attempt_timeout: Duration::from_millis(80),
retry_interval: Duration::from_millis(10),
startup_deadline: Duration::from_millis(100),
}
}

#[tokio::test(start_paused = true)]
async fn failed_connection_retries_then_pool_contains_every_slot() {
let attempts = Cell::new(0);
let started = Instant::now();
let channels = connect_pool_with("test", "in-memory", config(), false, |slot, timeout| {
assert_eq!(timeout, Duration::from_millis(80));
let attempt = attempts.get();
attempts.set(attempt + 1);
async move {
if attempt == 0 {
Err(io::Error::other("initial failure"))
} else {
Ok(slot)
}
}
})
.await
.unwrap();
assert_eq!(channels, vec![1, 2, 3]);
assert_eq!(attempts.get(), 4);
assert_eq!(started.elapsed(), Duration::from_millis(10));
}

#[tokio::test(start_paused = true)]
async fn later_pool_slots_share_the_original_deadline() {
let observed = std::cell::RefCell::new(Vec::new());
let started = Instant::now();
let error = connect_pool_with("test", "in-memory", config(), false, |slot, timeout| {
observed.borrow_mut().push((slot, timeout));
async move {
if slot == 1 {
sleep(Duration::from_millis(70)).await;
Ok(slot)
} else {
std::future::pending::<Result<usize, io::Error>>().await
}
}
})
.await
.unwrap_err();
assert_eq!(started.elapsed(), Duration::from_millis(100));
assert_eq!(
*observed.borrow(),
vec![
(1, Duration::from_millis(80)),
(2, Duration::from_millis(30)),
(3, Duration::from_millis(30))
]
);
assert_eq!(
error.error_type(),
ErrorType::Backend(BackendError::CannotConnect)
);
assert!(error.to_string().contains("pool slot 2"));
assert!(error.to_string().contains("exceeded the startup deadline"));
}

#[tokio::test(start_paused = true)]
async fn retry_wait_is_capped_and_last_failure_is_preserved() {
Comment thread
JulienDarve marked this conversation as resolved.
Resolved
let mut transport = config();
transport.retry_interval = Duration::from_secs(1);
let started = Instant::now();
let error = connect_pool_with("test", "in-memory", transport, false, |_, _| async {
Err::<(), _>(io::Error::other("peer rejected connection"))
})
.await
.unwrap_err();
assert_eq!(started.elapsed(), Duration::from_millis(100));
assert!(error.to_string().contains("peer rejected connection"));
assert!(error.to_string().contains("after 1 attempts"));
}
Loading
Loading