Add metrics representing silo quota limits and utilization.

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -61,6 +61,7 @@ use nexus_types::internal_api::background::AuditLogTimeoutIncompleteStatus;
 use nexus_types::internal_api::background::BlueprintPlannerStatus;
 use nexus_types::internal_api::background::BlueprintPrunerStatus;
 use nexus_types::internal_api::background::BlueprintRendezvousStatus;
+use nexus_types::internal_api::background::CapacityMetricsStatus;
 use nexus_types::internal_api::background::DatasetRendezvousOutcome;
 use nexus_types::internal_api::background::DatasetRendezvousStats;
 use nexus_types::internal_api::background::DatasetsRendezvousStats;
@@ -1341,6 +1342,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "instance_watcher" => {
             print_task_instance_watcher(details);
         }
+        "capacity_metrics" => {
+            print_task_capacity_metrics(details);
+        }
         "inventory_collection" => {
             print_task_inventory_collection(details);
         }
@@ -2232,6 +2236,17 @@ fn print_task_instance_updater(details: &serde_json::Value) {
     println!("    {SAGA_ERRORS:<WIDTH$}{:>NUM_WIDTH$}", saga_errors.len());
 }
 
+fn print_task_capacity_metrics(details: &serde_json::Value) {
+    match serde_json::from_value::<CapacityMetricsStatus>(details.clone()) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {error:?}: {details:?}",
+        ),
+        Ok(status) => {
+            println!("    samples queued: {}", status.samples_queued);
+        }
+    }
+}
+
 fn print_task_instance_watcher(details: &serde_json::Value) {
     #[derive(Deserialize)]
     struct TaskSuccess {

dev-tools/omdb/tests/env.out

@@ -68,6 +68,10 @@ task: "blueprint_rendezvous"
     owned rendezvous tables that other subsystems consume
 
 
+task: "capacity_metrics"
+    periodically emits quota capacity metrics
+
+
 task: "crdb_node_id_collector"
     Collects node IDs of running CockroachDB zones
 
@@ -344,6 +348,10 @@ task: "blueprint_rendezvous"
     owned rendezvous tables that other subsystems consume
 
 
+task: "capacity_metrics"
+    periodically emits quota capacity metrics
+
+
 task: "crdb_node_id_collector"
     Collects node IDs of running CockroachDB zones
 
@@ -607,6 +615,10 @@ task: "blueprint_rendezvous"
     owned rendezvous tables that other subsystems consume
 
 
+task: "capacity_metrics"
+    periodically emits quota capacity metrics
+
+
 task: "crdb_node_id_collector"
     Collects node IDs of running CockroachDB zones
 

dev-tools/omdb/tests/successes.out

@@ -398,6 +398,10 @@ task: "blueprint_rendezvous"
     owned rendezvous tables that other subsystems consume
 
 
+task: "capacity_metrics"
+    periodically emits quota capacity metrics
+
+
 task: "crdb_node_id_collector"
     Collects node IDs of running CockroachDB zones
 
@@ -780,6 +784,12 @@ task: "blueprint_rendezvous"
         num_not_in_blueprint:                0
         num_decommissioned_not_in_blueprint: 0
 
+task: "capacity_metrics"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    samples queued: 0
+
 task: "crdb_node_id_collector"
   configured period: every <REDACTED_DURATION>m
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1518,6 +1528,12 @@ task: "blueprint_rendezvous"
         num_not_in_blueprint:                0
         num_decommissioned_not_in_blueprint: 0
 
+task: "capacity_metrics"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    samples queued: 0
+
 task: "crdb_node_id_collector"
   configured period: every <REDACTED_DURATION>m
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

nexus-config/src/nexus_config.rs

@@ -447,6 +447,8 @@ pub struct BackgroundTaskConfig {
     pub region_replacement_driver: RegionReplacementDriverConfig,
     /// configuration for instance watcher task
     pub instance_watcher: InstanceWatcherConfig,
+    /// configuration for capacity metrics task
+    pub capacity_metrics: CapacityMetricsConfig,
     /// configuration for instance updater task
     pub instance_updater: InstanceUpdaterConfig,
     /// configuration for instance reincarnation task
@@ -767,6 +769,14 @@ pub struct InstanceWatcherConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct CapacityMetricsConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct InstanceUpdaterConfig {
@@ -1381,6 +1391,7 @@ mod test {
             region_replacement.period_secs = 30
             region_replacement_driver.period_secs = 30
             instance_watcher.period_secs = 30
+            capacity_metrics.period_secs = 60
             instance_updater.period_secs = 30
             instance_updater.disable = false
             instance_reincarnation.period_secs = 67
@@ -1609,6 +1620,9 @@ mod test {
                         instance_watcher: InstanceWatcherConfig {
                             period_secs: Duration::from_secs(30),
                         },
+                        capacity_metrics: CapacityMetricsConfig {
+                            period_secs: Duration::from_secs(60),
+                        },
                         instance_updater: InstanceUpdaterConfig {
                             period_secs: Duration::from_secs(30),
                             disable: false,
@@ -1798,6 +1812,7 @@ mod test {
             region_replacement.period_secs = 30
             region_replacement_driver.period_secs = 30
             instance_watcher.period_secs = 30
+            capacity_metrics.period_secs = 60
             instance_updater.period_secs = 30
             instance_reincarnation.period_secs = 67
             service_firewall_propagation.period_secs = 300

nexus/background-task-interface/src/init.rs

@@ -32,6 +32,7 @@ pub struct BackgroundTasks {
     pub task_region_replacement: Activator,
     pub task_region_replacement_driver: Activator,
     pub task_instance_watcher: Activator,
+    pub task_capacity_metrics: Activator,
     pub task_instance_updater: Activator,
     pub task_instance_reincarnation: Activator,
     pub task_service_firewall_propagation: Activator,

nexus/examples/config-second.toml

@@ -155,6 +155,7 @@ region_replacement.period_secs = 30
 region_replacement_driver.period_secs = 30
 # How frequently to query the status of active instances.
 instance_watcher.period_secs = 30
+capacity_metrics.period_secs = 60
 # How frequently to schedule new instance update sagas.
 instance_updater.period_secs = 30
 # How frequently to attempt to restart Failed instances?

nexus/examples/config.toml

@@ -146,6 +146,7 @@ region_replacement.period_secs = 30
 region_replacement_driver.period_secs = 30
 # How frequently to query the status of active instances.
 instance_watcher.period_secs = 30
+capacity_metrics.period_secs = 60
 # How frequently to schedule new instance update sagas.
 instance_updater.period_secs = 30
 # How frequently to attempt to restart Failed instances?

nexus/src/app/background/init.rs

@@ -100,6 +100,7 @@ use super::tasks::blueprint_load::LoadedTargetBlueprint;
 use super::tasks::blueprint_planner;
 use super::tasks::blueprint_pruner;
 use super::tasks::blueprint_rendezvous;
+use super::tasks::capacity_metrics;
 use super::tasks::crdb_node_id_collector;
 use super::tasks::decommissioned_disk_cleaner;
 use super::tasks::dns_config;
@@ -250,6 +251,7 @@ impl BackgroundTasksInitializer {
             task_region_replacement: Activator::new(),
             task_region_replacement_driver: Activator::new(),
             task_instance_watcher: Activator::new(),
+            task_capacity_metrics: Activator::new(),
             task_instance_updater: Activator::new(),
             task_instance_reincarnation: Activator::new(),
             task_service_firewall_propagation: Activator::new(),
@@ -350,6 +352,7 @@ impl BackgroundTasksInitializer {
             task_region_replacement_driver,
             task_instance_watcher,
             task_instance_updater,
+            task_capacity_metrics,
             task_instance_reincarnation,
             task_service_firewall_propagation,
             task_abandoned_vmm_reaper,
@@ -808,6 +811,23 @@ impl BackgroundTasksInitializer {
             })
         };
 
+        {
+            let capacity_metrics = capacity_metrics::CapacityMetrics::new(
+                datastore.clone(),
+                producer_registry,
+                nexus_id,
+            );
+            driver.register(TaskDefinition {
+                name: "capacity_metrics",
+                description: "periodically emits quota capacity metrics",
+                period: config.capacity_metrics.period_secs,
+                task_impl: Box::new(capacity_metrics),
+                opctx: opctx.child(BTreeMap::new()),
+                watchers: vec![],
+                activator: task_capacity_metrics,
+            });
+        }
+
         // Background task: schedule update sagas for instances in need of
         // state updates.
         {