Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ See [Forward Pass Metrics Trace Reference](forward-pass-metrics-traces.mdx) for
</ParamField>

<ParamField path="DYN_REQUEST_TRACE_S3_REGION" type="string">
Region override for the `s3` sink. When unset the AWS SDK resolves the region from the environment, profile, or IMDS.
Region override for the `s3` sink. When unset, the client uses `AWS_REGION`, then `AWS_DEFAULT_REGION`, then `us-east-1`.
</ParamField>

<ParamField path="DYN_REQUEST_TRACE_S3_PREFIX" type="string">
Expand Down
8 changes: 4 additions & 4 deletions docs/fern/pages/reference/observability/request-traces.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ batch. Records are batched in-process and uploaded when the batch reaches
`DYN_REQUEST_TRACE_S3_ROLL_UNCOMPRESSED_BYTES` or when `DYN_REQUEST_TRACE_S3_FLUSH_INTERVAL_MS`
elapses. Object keys are `{prefix}/{yyyy}/{mm}/{dd}/{host}-{HHMMSS}-{run_id}-{seq}.jsonl.gz`, where
`run_id` is a per-process UUID that keeps container restarts and hostname collisions from overwriting
earlier batches. Credentials come from the AWS SDK default provider chain (environment variables,
IMDS, IRSA, Pod Identity, shared profiles); how the frontend pod is credentialed is a deployment
concern. On terminal upload failure the batch is dropped and a warning is logged; no on-disk retry
queue is kept.
earlier batches. Credentials come from the shared object-store client's provider chain (environment
variables, IMDS, IRSA, and Pod Identity). Shared AWS config and credential profiles are not loaded;
how the frontend pod is credentialed is a deployment concern. On terminal upload failure the batch
is dropped and a warning is logged; no on-disk retry queue is kept.

```bash
export DYN_REQUEST_TRACE=1
Expand Down
2 changes: 0 additions & 2 deletions lib/bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions lib/bindings/python/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ nvtx = ["dynamo-runtime/nvtx"]
# recipes pass this flag explicitly; the dev container's
# `.devcontainer/post-create.sh` also enables it for local workflows.
mm-routing = ["dynamo-llm/mm-routing"]
# Enable the native S3 destination for the request-trace sink. Pulls in
# aws-sdk-s3 + aws-config so the default wheel stays lean; production wheel
# Enable the native S3 destination for the request-trace sink. Production wheel
# builds enable this flag in container/templates/wheel_builder.Dockerfile so
# `DYN_REQUEST_TRACE_SINKS=s3` works on shipped artifacts.
request-trace-s3 = ["dynamo-llm/request-trace-s3"]
Expand Down
8 changes: 2 additions & 6 deletions lib/llm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ ckf-diagnostics = []
kv-router-stress = ["dep:clap", "dep:indicatif", "bench"]
mm-routing = ["dep:llm-multimodal", "dep:llm-tokenizer"]
request-trace-bench = []
# S3 destination for the request-trace sink. Pulls in aws-sdk-s3 + aws-config.
request-trace-s3 = ["dep:aws-sdk-s3", "dep:aws-config"]
# S3 destination for the request-trace sink, reusing the existing object_store client.
request-trace-s3 = []

[[bench]]
name = "tokenizer_simple"
Expand Down Expand Up @@ -144,10 +144,6 @@ nix = { version = "0.26", optional = true }
# media (zlib compression for NIXL metadata)
flate2 = { version = "1" }

# request-trace-s3 (optional S3 sink)
aws-sdk-s3 = { version = "1.120.0", optional = true }
aws-config = { version = "1.8.11", optional = true }

# block_manager_bench
clap = { version = "4.5.49", features = ["derive"], optional = true }
indicatif = { version = "0.18.0", optional = true }
Expand Down
191 changes: 129 additions & 62 deletions lib/llm/src/request_trace/s3_sink.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
//! (`{prefix}/{yyyy}/{mm}/{dd}/{host}-{HHMMSS}-{run_id}-{seq}.jsonl.gz`);
//! richer partitioning ships in a follow-up.
//!
//! Credentials come from the AWS SDK default provider chain — env vars, IMDS,
//! IRSA, Pod Identity, and shared profiles are all handled by the SDK. How the
//! frontend pod is credentialed is a deployment concern, not this sink's.
//! Credentials come from the shared `object_store` S3 client — environment
//! variables, IMDS, IRSA, and Pod Identity are supported. Shared AWS config and
//! credential profiles are not loaded; how the frontend pod is credentialed is
//! a deployment concern, not this sink's.
Comment thread
PeaBrane marked this conversation as resolved.
Resolved
//!
//! # Upload concurrency
//!
Expand All @@ -31,9 +32,12 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};

use anyhow::{Context as _, Result};
use async_trait::async_trait;
use aws_config::{BehaviorVersion, Region, timeout::TimeoutConfig};
use aws_sdk_s3::primitives::ByteStream;
use flate2::{Compression, write::GzEncoder};
use object_store::{
Attribute, Attributes, ClientConfigKey, ObjectStore, RetryConfig,
aws::{AmazonS3Builder, AmazonS3ConfigKey},
path::Path as ObjectPath,
};
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
Expand All @@ -48,8 +52,8 @@ const CHANNEL_CAPACITY: usize = 2048;
const DEFAULT_BUFFER_INITIAL_BYTES: usize = 256 * 1024;
// Bound S3 upload duration so a stalled endpoint or slow network cannot wedge
// the worker task indefinitely. `attempt_timeout` covers a single HTTP attempt;
// `operation_timeout` bounds the full call including SDK retries (three total
// by default). After the operation timeout expires the batch is discarded with
// the outer timeout bounds the full call including retries (three total).
// After the operation timeout expires the batch is discarded with
// a warning; a persistent retry queue is deferred to a follow-up PR.
const S3_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30);
const S3_OPERATION_TIMEOUT: Duration = Duration::from_secs(90);
Expand Down Expand Up @@ -91,17 +95,11 @@ impl S3RequestTraceSink {
let roll_uncompressed_bytes = policy.s3_roll_uncompressed_bytes;
let flush_interval = Duration::from_millis(policy.s3_flush_interval_ms.max(1));

let timeout_config = TimeoutConfig::builder()
.operation_attempt_timeout(S3_ATTEMPT_TIMEOUT)
.operation_timeout(S3_OPERATION_TIMEOUT)
.build();
let mut loader =
aws_config::defaults(BehaviorVersion::latest()).timeout_config(timeout_config);
if let Some(region) = policy.s3_region.clone() {
loader = loader.region(Region::new(region));
}
let sdk_config = loader.load().await;
let client = aws_sdk_s3::Client::new(&sdk_config);
let store: Arc<dyn ObjectStore> = Arc::new(
request_trace_s3_builder(&bucket, policy.s3_region.as_deref())
.build()
.context("building request trace S3 client")?,
);

let (tx, rx) = mpsc::channel(CHANNEL_CAPACITY);
let shutdown = CancellationToken::new();
Expand All @@ -114,7 +112,7 @@ impl S3RequestTraceSink {
let worker_shutdown = shutdown.clone();
let worker = tokio::spawn(async move {
run_worker(
client,
store,
upload_options,
rx,
worker_shutdown,
Expand Down Expand Up @@ -200,14 +198,14 @@ impl RequestTraceSink for S3RequestTraceSink {
}

async fn run_worker(
client: aws_sdk_s3::Client,
store: Arc<dyn ObjectStore>,
options: S3UploadOptions,
mut rx: mpsc::Receiver<RequestTraceRecord>,
shutdown: CancellationToken,
roll_uncompressed_bytes: u64,
flush_interval: Duration,
) {
let uploader = Arc::new(S3Uploader { client, options });
let uploader = Arc::new(S3Uploader { store, options });
let mut batch = JsonlBatch::new();
let mut seq: u64 = 0;
let mut flush_tick = tokio::time::interval(flush_interval);
Expand Down Expand Up @@ -289,22 +287,22 @@ async fn upload_ready_batch(uploader: &Arc<S3Uploader>, batch: &mut JsonlBatch,
let key = uploader.object_key(SystemTime::now(), this_seq);
let batch_bytes = ready.len();
if let Err(error) = uploader.put_object(key.clone(), ready).await {
// The SDK exhausted its retries (three total attempts by default,
// bounded by the operation timeout). The batch is dropped here rather
// The client exhausted its retries (three total attempts, bounded by
// the operation timeout). The batch is dropped here rather
// than requeued; a persistent retry buffer is a follow-up concern
// tracked in the S3 layout PR.
tracing::warn!(
target: "dynamo_llm::request_trace",
key = %key,
batch_bytes,
%error,
"request trace s3: put_object failed after SDK retries; batch discarded"
"request trace s3: put_object failed after retries; batch discarded"
);
}
}

struct S3Uploader {
client: aws_sdk_s3::Client,
store: Arc<dyn ObjectStore>,
options: S3UploadOptions,
}

Expand Down Expand Up @@ -336,19 +334,46 @@ impl S3Uploader {
}

async fn put_object(&self, key: String, body: Vec<u8>) -> Result<()> {
self.client
.put_object()
.bucket(&self.options.bucket)
.key(key)
.content_type("application/gzip")
.body(ByteStream::from(body))
.send()
.await
.with_context(|| format!("s3 put_object into bucket {}", self.options.bucket))?;
let location = ObjectPath::from(key);
Comment thread
PeaBrane marked this conversation as resolved.
Resolved
let attributes = Attributes::from_iter([(Attribute::ContentType, "application/gzip")]);
tokio::time::timeout(
S3_OPERATION_TIMEOUT,
self.store
.put_opts(&location, body.into(), attributes.into()),
)
.await
.with_context(|| {
format!(
"s3 put_object into bucket {} timed out",
self.options.bucket
)
})?
.with_context(|| format!("s3 put_object into bucket {}", self.options.bucket))?;
Ok(())
}
}

fn request_trace_s3_builder(bucket: &str, region: Option<&str>) -> AmazonS3Builder {
let retry_config = RetryConfig {
max_retries: 2,
retry_timeout: S3_OPERATION_TIMEOUT,
..Default::default()
};
let mut builder = AmazonS3Builder::from_env()
.with_bucket_name(bucket)
.with_config(
AmazonS3ConfigKey::Client(ClientConfigKey::Timeout),
format!("{}s", S3_ATTEMPT_TIMEOUT.as_secs()),
)
.with_retry(retry_config);
if let Some(region) = region {
builder = builder.with_region(region);
} else if let Ok(region) = std::env::var("AWS_REGION") {
builder = builder.with_region(region);
}
builder
}

/// Buffers raw JSONL bytes until the batch is ready to flush; gzip
/// compression happens in [`take_finished`], which offloads the CPU work
/// to a blocking pool (matching `telemetry::jsonl_gz`).
Expand Down Expand Up @@ -446,6 +471,7 @@ fn utc_date_parts(secs: u64) -> (i64, u32, u32, u32, u32, u32) {
mod tests {
use super::*;
use crate::request_trace::{RequestTraceEventType, RequestTraceSchema};
use object_store::memory::InMemory;

#[test]
fn utc_date_parts_epoch() {
Expand All @@ -462,15 +488,7 @@ mod tests {

#[test]
fn object_key_includes_prefix_date_run_and_seq() {
let uploader = S3Uploader {
client: dummy_client(),
options: S3UploadOptions {
bucket: "b".to_string(),
prefix: "traces/".to_string(),
host: "frontend-0".to_string(),
run_id: "cafebabe".to_string(),
},
};
let uploader = test_uploader("traces/");
let at = UNIX_EPOCH + Duration::from_secs(1_784_118_896);
let key = uploader.object_key(at, 42);
assert_eq!(
Expand All @@ -481,34 +499,83 @@ mod tests {

#[test]
fn object_key_omits_leading_slash_when_prefix_empty() {
let uploader = S3Uploader {
client: dummy_client(),
options: S3UploadOptions {
bucket: "b".to_string(),
prefix: String::new(),
host: "h".to_string(),
run_id: "abc".to_string(),
},
};
let uploader = test_uploader("");
let key = uploader.object_key(UNIX_EPOCH, 0);
assert!(!key.starts_with('/'));
assert!(key.starts_with("1970/01/01/h-"));
assert!(key.starts_with("1970/01/01/frontend-0-"));
}

#[tokio::test]
async fn put_object_writes_body_and_content_type() {
let store = Arc::new(InMemory::new());
let uploader = S3Uploader {
store: store.clone(),
options: test_options(""),
};
let key = "traces/batch.jsonl.gz";
let body = vec![1, 2, 3, 4];

uploader
.put_object(key.to_string(), body.clone())
.await
.unwrap();

let result = store.get(&ObjectPath::from(key)).await.unwrap();
assert_eq!(
result
.attributes
.get(&Attribute::ContentType)
.map(AsRef::as_ref),
Some("application/gzip")
);
assert_eq!(result.bytes().await.unwrap().as_ref(), body.as_slice());
}

fn dummy_client() -> aws_sdk_s3::Client {
// A no-network client. Any subsequent `send()` call would fail, but
// pure-Rust key generation does not exercise the transport.
let config = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.build();
aws_sdk_s3::Client::from_conf(config)
#[test]
fn s3_builder_preserves_environment_client_options() {
temp_env::with_vars(
[
("AWS_ALLOW_HTTP", Some("true")),
("AWS_PROXY_URL", Some("http://proxy.example:8080")),
],
|| {
let builder = request_trace_s3_builder("b", Some("us-west-2"));
assert_eq!(
builder
.get_config_value(&AmazonS3ConfigKey::Client(ClientConfigKey::AllowHttp))
.as_deref(),
Some("true")
);
assert_eq!(
builder
.get_config_value(&AmazonS3ConfigKey::Client(ClientConfigKey::ProxyUrl))
.as_deref(),
Some("http://proxy.example:8080")
);
},
);
}

fn test_uploader(prefix: &str) -> S3Uploader {
S3Uploader {
store: Arc::new(InMemory::new()),
options: test_options(prefix),
}
}

fn test_options(prefix: &str) -> S3UploadOptions {
S3UploadOptions {
bucket: "b".to_string(),
prefix: prefix.to_string(),
host: "frontend-0".to_string(),
run_id: "cafebabe".to_string(),
}
}

/// Build a sink whose bounded channel is never drained, so `emit` hits the
/// same backpressure path a stalled uploader would cause. The receiver is
/// returned to the caller and held so the channel stays open (full), not
/// closed. No AWS client or worker task is created.
/// closed. No S3 client or worker task is created.
fn stalled_sink(capacity: usize) -> (S3RequestTraceSink, mpsc::Receiver<RequestTraceRecord>) {
let (tx, rx) = mpsc::channel(capacity);
let sink = S3RequestTraceSink {
Expand Down
Loading