Skip to content

Commit 5bb473f

Browse files
Turbo87Turbo87
authored
cdn_logs: Add test for Fastly log file with extended metadata (#12209)
This adds a `test_full_info()` test case with anonymized test data to verify parsing of Fastly logs with extended field information like user agent headers.
1 parent 9d6e1e5 commit 5bb473f

File tree

2 files changed

+53
-0
lines changed

2 files changed

+53
-0
lines changed

‎crates/crates_io_cdn_logs/src/fastly/mod.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,4 +171,37 @@ mod tests {
171171
}
172172
");
173173
}
174+
175+
#[tokio::test]
176+
async fn test_full_info() {
177+
let _guard = enable_tracing_output();
178+
179+
let mut cursor = Cursor::new(include_bytes!("../../test_data/fastly/full-info.log"));
180+
let downloads = assert_ok!(count_downloads(&mut cursor).await);
181+
182+
assert_debug_snapshot!(downloads, @r"
183+
DownloadsMap {
184+
2025-10-26 [email protected] .. 1
185+
2025-10-26 [email protected] .. 1
186+
2025-10-26 [email protected] .. 1
187+
2025-10-26 [email protected] .. 1
188+
2025-10-26 [email protected] .. 1
189+
2025-10-26 [email protected] .. 1
190+
2025-10-26 [email protected] .. 1
191+
2025-10-26 [email protected] .. 1
192+
2025-10-26 [email protected] .. 1
193+
2025-10-26 [email protected] .. 1
194+
2025-10-26 [email protected] .. 1
195+
2025-10-26 [email protected] .. 1
196+
2025-10-26 [email protected] .. 1
197+
2025-10-26 [email protected] .. 1
198+
2025-10-26 [email protected] .. 1
199+
2025-10-26 [email protected] .. 1
200+
2025-10-26 [email protected] .. 1
201+
2025-10-26 [email protected] .. 1
202+
2025-10-26 [email protected] .. 1
203+
2025-10-26 [email protected] .. 1
204+
}
205+
");
206+
}
174207
}

‎crates/crates_io_cdn_logs/test_data/fastly/full-info.log‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<134>2025-10-26T23:57:34Z cache-chi-kigq8000030 s3-request-logs[257678]: {"bytes":36308,"content_type":"application/gzip","date_time":"2025-10-26T23:57:34.867635728Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"CHI","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.92.0-nightly (344c4567c 2025-10-21)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/scale-info/2.11.3/download","version":"1"}
2+
<134>2025-10-26T23:57:34Z cache-bfi-kbfi7400083 s3-request-logs[257678]: {"bytes":87113,"content_type":"application/x-tar","date_time":"2025-10-26T23:57:34.892608199Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"BFI","host":"static.crates.io","http":{"protocol":"HTTP/1.1","referer":null,"useragent":"Bazel/release 7.6.2"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/rand/0.8.5/download","version":"1"}
3+
<134>2025-10-26T23:57:34Z cache-wsi-ysbk1060052 s3-request-logs[257678]: {"bytes":24828,"content_type":"application/gzip","date_time":"2025-10-26T23:57:34.776171793Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"WSI","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.88.0 (873a06493 2025-05-10)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/dashmap/6.1.0/download","version":"1"}
4+
<134>2025-10-26T23:57:35Z cache-gru-sbsp2090059 s3-request-logs[257678]: {"bytes":5091,"content_type":"application/gzip","date_time":"2025-10-26T23:57:34.155889446Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"GRU","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.88.0 (873a06493 2025-05-10)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/cargo-set-version/0.0.2/download","version":"1"}
5+
<134>2025-10-26T23:57:35Z cache-bfi-kbfi7400025 s3-request-logs[257678]: {"bytes":49331,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.076639302Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"BFI","host":"static.crates.io","http":{"protocol":"HTTP/1.1","referer":null,"useragent":"Bazel/release 7.6.2"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/matrixmultiply/0.3.3/download","version":"1"}
6+
<134>2025-10-26T23:57:35Z cache-bfi-kbfi7400035 s3-request-logs[257678]: {"bytes":109417,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.164124132Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"BFI","host":"static.crates.io","http":{"protocol":"HTTP/1.1","referer":null,"useragent":"Bazel/release 7.6.2"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/tower/0.5.2/download","version":"1"}
7+
<134>2025-10-26T23:57:35Z cache-sin-wsat1880044 s3-request-logs[257678]: {"bytes":284789,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.315876169Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"SIN","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/precis-profiles/0.1.11/download","version":"1"}
8+
<134>2025-10-26T23:57:35Z cache-sin-wsat1880044 s3-request-logs[257678]: {"bytes":122604,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.316427427Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"SIN","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/unicode-normalization/0.1.22/download","version":"1"}
9+
<134>2025-10-26T23:57:35Z cache-sin-wsat1880044 s3-request-logs[257678]: {"bytes":5865,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.357187387Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"SIN","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/tinyvec_macros/0.1.1/download","version":"1"}
10+
<134>2025-10-26T23:57:35Z cache-sin-wsat1880044 s3-request-logs[257678]: {"bytes":18278,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.390314629Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"SIN","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/precis-tools/0.1.8/download","version":"1"}
11+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000167 s3-request-logs[257678]: {"bytes":6125,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.639093858Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/http-body/1.0.1/download","version":"1"}
12+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":26114,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.653539031Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/gix-packetline/0.19.3/download","version":"1"}
13+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":792045,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.670041691Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/libc/0.2.177/download","version":"1"}
14+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":106063,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.670818768Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/http/1.3.1/download","version":"1"}
15+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":46735,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.667106366Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/parking_lot/0.12.5/download","version":"1"}
16+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":29718,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.676733818Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/ipnet/2.11.0/download","version":"1"}
17+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":22568,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.676056602Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/gix-refspec/0.30.1/download","version":"1"}
18+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":9090,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.672179519Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/lru-slab/0.1.2/download","version":"1"}
19+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":38315,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.667280377Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/owo-colors/4.2.3/download","version":"1"}
20+
<134>2025-10-26T23:57:35Z cache-iad-kiad7000077 s3-request-logs[257678]: {"bytes":100446,"content_type":"application/gzip","date_time":"2025-10-26T23:57:35.674317517Z","ddsource":"fastly","ddtags":"app:crates.io,env:prod","edge_location":"IAD","host":"static.crates.io","http":{"protocol":"HTTP/2","referer":null,"useragent":"cargo/1.90.0 (840b83a10 2025-07-30)"},"ip":"192.0.2.1","method":"GET","service":"static.crates.io","status":200,"tls":{"cipher":"TLS_AES_128_GCM_SHA256","protocol":"TLSv1.3"},"url":"https://static.crates.io/crates/indexmap/2.12.0/download","version":"1"}

0 commit comments

Comments
 (0)