Repository navigation
test failed in CI: omicron-clickhouse-admin::context::tests::test_database_usage #10398
Description
This test failed on a CI run on pull request #10389:
Log showing the specific test failure:
Excerpt from the log showing the failure:
8269 2026-05-06T22:27:46.109Z TRY 1 FAIL [ 1.181s] (─────────) omicron-clickhouse-admin context::tests::test_database_usage
8270 2026-05-06T22:27:46.109Z stdout ───
8271 2026-05-06T22:27:46.109Z
8272 2026-05-06T22:27:46.109Z running 1 test
8273 2026-05-06T22:27:46.109Z DatabaseUsageResult {
8274 2026-05-06T22:27:46.109Z last_success: Some(
8275 2026-05-06T22:27:46.109Z DatabaseUsage {
8276 2026-05-06T22:27:46.109Z tables: {},
8277 2026-05-06T22:27:46.109Z started_at: 2026-05-06T22:27:44.173282499Z,
8278 2026-05-06T22:27:46.110Z completed_at: 2026-05-06T22:27:44.195073479Z,
8279 2026-05-06T22:27:46.110Z },
8280 2026-05-06T22:27:46.110Z ),
8281 2026-05-06T22:27:46.110Z last_error: None,
8282 2026-05-06T22:27:46.110Z }
8283 2026-05-06T22:27:46.110Z DatabaseUsageResult {
8284 2026-05-06T22:27:46.110Z last_success: Some(
8285 2026-05-06T22:27:46.110Z DatabaseUsage {
8286 2026-05-06T22:27:46.110Z tables: {},
8287 2026-05-06T22:27:46.110Z started_at: 2026-05-06T22:27:44.173282499Z,
8288 2026-05-06T22:27:46.110Z completed_at: 2026-05-06T22:27:44.195073479Z,
8289 2026-05-06T22:27:46.110Z },
8290 2026-05-06T22:27:46.110Z ),
8291 2026-05-06T22:27:46.110Z last_error: Some(
8292 2026-05-06T22:27:46.110Z DatabaseUsageError {
8293 2026-05-06T22:27:46.110Z timestamp: 2026-05-06T22:27:44.529983057Z,
8294 2026-05-06T22:27:46.110Z error: "Telemetry database unavailable: SQL query timed out after 30.02s",
8295 2026-05-06T22:27:46.110Z },
8296 2026-05-06T22:27:46.110Z ),
8297 2026-05-06T22:27:46.110Z }
8298 2026-05-06T22:27:46.110Z test context::tests::test_database_usage ... FAILED
8299 2026-05-06T22:27:46.110Z
8300 2026-05-06T22:27:46.110Z failures:
8301 2026-05-06T22:27:46.110Z
8302 2026-05-06T22:27:46.110Z failures:
8303 2026-05-06T22:27:46.110Z context::tests::test_database_usage
8304 2026-05-06T22:27:46.110Z
8305 2026-05-06T22:27:46.110Z test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 6 filtered out; finished in 1.12s
8306 2026-05-06T22:27:46.110Z
8307 2026-05-06T22:27:46.110Z stderr ───
8308 2026-05-06T22:27:46.110Z log file: /var/tmp/omicron_tmp/omicron_clickhouse_admin-5d4d58c20667040a-test_database_usage.18344.0.log
8309 2026-05-06T22:27:46.110Z note: configured to log to "/var/tmp/omicron_tmp/omicron_clickhouse_admin-5d4d58c20667040a-test_database_usage.18344.0.log"
8310 2026-05-06T22:27:46.110Z
8311 2026-05-06T22:27:46.110Z thread 'context::tests::test_database_usage' (2) panicked at clickhouse-admin/src/context.rs:972:69:
8312 2026-05-06T22:27:46.110Z called `Option::unwrap()` on a `None` value
8313 2026-05-06T22:27:46.110Z stack backtrace:
8314 2026-05-06T22:27:46.110Z 0: __rustc::rust_begin_unwind
8315 2026-05-06T22:27:46.110Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/panicking.rs:689:5
8316 2026-05-06T22:27:46.111Z 1: core::panicking::panic_fmt
8317 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:80:14
8318 2026-05-06T22:27:46.111Z 2: core::panicking::panic
8319 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/panicking.rs:150:5
8320 2026-05-06T22:27:46.111Z 3: core::option::unwrap_failed
8321 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/option.rs:2199:5
8322 2026-05-06T22:27:46.111Z 4: unwrap<&clickhouse_admin_types_versions::v2::usage::TableUsage>
8323 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/option.rs:1016:21
8324 2026-05-06T22:27:46.111Z 5: {async_block#0}
8325 2026-05-06T22:27:46.111Z at ./src/context.rs:972:69
8326 2026-05-06T22:27:46.111Z 6: poll<&mut dyn core::future::future::Future<Output=()>>
8327 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9
8328 2026-05-06T22:27:46.111Z 7: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
8329 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/future/future.rs:133:9
8330 2026-05-06T22:27:46.111Z 8: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
8331 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:70
8332 2026-05-06T22:27:46.111Z 9: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
8333 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:167:5
8334 2026-05-06T22:27:46.111Z 10: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
8335 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/task/coop/mod.rs:133:5
8336 2026-05-06T22:27:46.111Z 11: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
8337 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:778:25
8338 2026-05-06T22:27:46.111Z 12: <tokio::runtime::scheduler::current_thread::Context>::enter::<core::task::poll::Poll<()>, <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}::{closure#0}>
8339 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:451:19
8340 2026-05-06T22:27:46.111Z 13: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
8341 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:777:44
8342 2026-05-06T22:27:46.111Z 14: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}
8343 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:68
8344 2026-05-06T22:27:46.111Z 15: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<<tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)>
8345 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/scoped.rs:40:9
8346 2026-05-06T22:27:46.111Z 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}
8347 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:38
8348 2026-05-06T22:27:46.111Z 17: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)>
8349 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:513:12
8350 2026-05-06T22:27:46.111Z 18: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>)>
8351 2026-05-06T22:27:46.111Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/std/src/thread/local.rs:477:20
8352 2026-05-06T22:27:46.111Z 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core>, core::option::Option<()>), <tokio::runtime::scheduler::current_thread::CoreGuard>::enter<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>::{closure#0}>
8353 2026-05-06T22:27:46.111Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context.rs:181:17
8354 2026-05-06T22:27:46.111Z 20: <tokio::runtime::scheduler::current_thread::CoreGuard>::enter::<<tokio::runtime::scheduler::current_thread::CoreGuard>::block_on<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>::{closure#0}, core::option::Option<()>>
8355 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:865:27
8356 2026-05-06T22:27:46.112Z 21: <tokio::runtime::scheduler::current_thread::CoreGuard>::block_on::<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>>
8357 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:765:24
8358 2026-05-06T22:27:46.112Z 22: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
8359 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:205:33
8360 2026-05-06T22:27:46.112Z 23: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::scheduler::current_thread::CurrentThread>::block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>::{closure#0}, ()>
8361 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/context/runtime.rs:65:16
8362 2026-05-06T22:27:46.112Z 24: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
8363 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/scheduler/current_thread/mod.rs:193:9
8364 2026-05-06T22:27:46.112Z 25: <tokio::runtime::runtime::Runtime>::block_on_inner::<core::pin::Pin<&mut dyn core::future::future::Future<Output = ()>>>
8365 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:371:52
8366 2026-05-06T22:27:46.112Z 26: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
8367 2026-05-06T22:27:46.112Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.52.1/src/runtime/runtime.rs:345:18
8368 2026-05-06T22:27:46.112Z 27: test_database_usage
8369 2026-05-06T22:27:46.116Z at ./src/context.rs:994:36
8370 2026-05-06T22:27:46.116Z 28: omicron_clickhouse_admin::context::tests::test_database_usage::{closure#0}
8371 2026-05-06T22:27:46.116Z at ./src/context.rs:909:35
8372 2026-05-06T22:27:46.116Z 29: <omicron_clickhouse_admin::context::tests::test_database_usage::{closure#0} as core::ops::function::FnOnce<()>>::call_once
8373 2026-05-06T22:27:46.116Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5
8374 2026-05-06T22:27:46.116Z 30: core::ops::function::FnOnce::call_once
8375 2026-05-06T22:27:46.116Z at /rustc/4a4ef493e3a1488c6e321570238084b38948f6db/library/core/src/ops/function.rs:250:5
8376 2026-05-06T22:27:46.116Z note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
8377 2026-05-06T22:27:46.116Z WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 18345) and a temporary directory leaked, /var/tmp/omicron_tmp/omicron_clickhouse_admin-5d4d58c20667040a-test_database_usage.18344.1-clickhouse-FB2lTj)
8378 2026-05-06T22:27:46.116Z failed to clean up ClickHouse data dir:
8379 2026-05-06T22:27:46.116Z - /var/tmp/omicron_tmp/omicron_clickhouse_admin-5d4d58c20667040a-test_database_usage.18344.1-clickhouse-FB2lTj: File exists (os error 17)
8380 2026-05-06T22:27:46.116Z
8381 2026-05-06T22:27:46.127Z PASS [ 10.368s] (1515/2819) omicron-nexus::test_all integration_tests::disks::test_disk_reject_total_size_greater_than_max_disk_size_bytes
8382 2026-05-06T22:27:47.066Z SLOW [> 60.000s] (─────────) omicron-nexus::test_all integration_tests::crucible_replacements::test_read_only_replacement_sanity
8383 2026-05-06T22:27:47.082Z TRY 2 PASS [ 0.974s] (1516/2819) omicron-clickhouse-admin context::tests::test_database_usage
8384 2026-05-06T22:27:47.082Z - test configured to fail if flaky
(note also the output from cargo nextest indicating that the test succeeded on the second try)
Activity
iliana commentedon May 12, 2026
Possibly seen on main: https://github.com/oxidecomputer/omicron/runs/75599594807
Don't manipulate time in oximeter database usage tests (#10405)