diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 9abe14a02ae..209a2f9246c 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -61,6 +61,7 @@ use nexus_types::internal_api::background::AuditLogTimeoutIncompleteStatus; use nexus_types::internal_api::background::BlueprintPlannerStatus; use nexus_types::internal_api::background::BlueprintPrunerStatus; use nexus_types::internal_api::background::BlueprintRendezvousStatus; +use nexus_types::internal_api::background::CapacityMetricsStatus; use nexus_types::internal_api::background::DatasetRendezvousOutcome; use nexus_types::internal_api::background::DatasetRendezvousStats; use nexus_types::internal_api::background::DatasetsRendezvousStats; @@ -1341,6 +1342,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { "instance_watcher" => { print_task_instance_watcher(details); } + "capacity_metrics" => { + print_task_capacity_metrics(details); + } "inventory_collection" => { print_task_inventory_collection(details); } @@ -2232,6 +2236,17 @@ fn print_task_instance_updater(details: &serde_json::Value) { println!(" {SAGA_ERRORS:NUM_WIDTH$}", saga_errors.len()); } +fn print_task_capacity_metrics(details: &serde_json::Value) { + match serde_json::from_value::(details.clone()) { + Err(error) => eprintln!( + "warning: failed to interpret task details: {error:?}: {details:?}", + ), + Ok(status) => { + println!(" samples queued: {}", status.samples_queued); + } + } +} + fn print_task_instance_watcher(details: &serde_json::Value) { #[derive(Deserialize)] struct TaskSuccess { diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 9ade7712ae8..a05fb443993 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -68,6 +68,10 @@ task: "blueprint_rendezvous" owned rendezvous tables that other subsystems consume +task: "capacity_metrics" + periodically emits quota capacity metrics + + task: "crdb_node_id_collector" Collects node IDs of running CockroachDB zones @@ -344,6 +348,10 @@ task: "blueprint_rendezvous" owned rendezvous tables that other subsystems consume +task: "capacity_metrics" + periodically emits quota capacity metrics + + task: "crdb_node_id_collector" Collects node IDs of running CockroachDB zones @@ -607,6 +615,10 @@ task: "blueprint_rendezvous" owned rendezvous tables that other subsystems consume +task: "capacity_metrics" + periodically emits quota capacity metrics + + task: "crdb_node_id_collector" Collects node IDs of running CockroachDB zones diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 4f8361fdcc9..a4519ef3fb0 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -398,6 +398,10 @@ task: "blueprint_rendezvous" owned rendezvous tables that other subsystems consume +task: "capacity_metrics" + periodically emits quota capacity metrics + + task: "crdb_node_id_collector" Collects node IDs of running CockroachDB zones @@ -780,6 +784,12 @@ task: "blueprint_rendezvous" num_not_in_blueprint: 0 num_decommissioned_not_in_blueprint: 0 +task: "capacity_metrics" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + samples queued: 0 + task: "crdb_node_id_collector" configured period: every m last completed activation: , triggered by @@ -1518,6 +1528,12 @@ task: "blueprint_rendezvous" num_not_in_blueprint: 0 num_decommissioned_not_in_blueprint: 0 +task: "capacity_metrics" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + samples queued: 0 + task: "crdb_node_id_collector" configured period: every m last completed activation: , triggered by diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 9bdfbbd9e90..47966376e35 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -447,6 +447,8 @@ pub struct BackgroundTaskConfig { pub region_replacement_driver: RegionReplacementDriverConfig, /// configuration for instance watcher task pub instance_watcher: InstanceWatcherConfig, + /// configuration for capacity metrics task + pub capacity_metrics: CapacityMetricsConfig, /// configuration for instance updater task pub instance_updater: InstanceUpdaterConfig, /// configuration for instance reincarnation task @@ -767,6 +769,14 @@ pub struct InstanceWatcherConfig { pub period_secs: Duration, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct CapacityMetricsConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + #[serde_as] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct InstanceUpdaterConfig { @@ -1381,6 +1391,7 @@ mod test { region_replacement.period_secs = 30 region_replacement_driver.period_secs = 30 instance_watcher.period_secs = 30 + capacity_metrics.period_secs = 60 instance_updater.period_secs = 30 instance_updater.disable = false instance_reincarnation.period_secs = 67 @@ -1609,6 +1620,9 @@ mod test { instance_watcher: InstanceWatcherConfig { period_secs: Duration::from_secs(30), }, + capacity_metrics: CapacityMetricsConfig { + period_secs: Duration::from_secs(60), + }, instance_updater: InstanceUpdaterConfig { period_secs: Duration::from_secs(30), disable: false, @@ -1798,6 +1812,7 @@ mod test { region_replacement.period_secs = 30 region_replacement_driver.period_secs = 30 instance_watcher.period_secs = 30 + capacity_metrics.period_secs = 60 instance_updater.period_secs = 30 instance_reincarnation.period_secs = 67 service_firewall_propagation.period_secs = 300 diff --git a/nexus/background-task-interface/src/init.rs b/nexus/background-task-interface/src/init.rs index b71458ca410..96c03ec31f1 100644 --- a/nexus/background-task-interface/src/init.rs +++ b/nexus/background-task-interface/src/init.rs @@ -32,6 +32,7 @@ pub struct BackgroundTasks { pub task_region_replacement: Activator, pub task_region_replacement_driver: Activator, pub task_instance_watcher: Activator, + pub task_capacity_metrics: Activator, pub task_instance_updater: Activator, pub task_instance_reincarnation: Activator, pub task_service_firewall_propagation: Activator, diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index baa812dc75d..1a1b57e6aed 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -155,6 +155,7 @@ region_replacement.period_secs = 30 region_replacement_driver.period_secs = 30 # How frequently to query the status of active instances. instance_watcher.period_secs = 30 +capacity_metrics.period_secs = 60 # How frequently to schedule new instance update sagas. instance_updater.period_secs = 30 # How frequently to attempt to restart Failed instances? diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 7fe5e6f5b6e..3523c827ea5 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -146,6 +146,7 @@ region_replacement.period_secs = 30 region_replacement_driver.period_secs = 30 # How frequently to query the status of active instances. instance_watcher.period_secs = 30 +capacity_metrics.period_secs = 60 # How frequently to schedule new instance update sagas. instance_updater.period_secs = 30 # How frequently to attempt to restart Failed instances? diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index edaf40d30e2..e571567a9f2 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -100,6 +100,7 @@ use super::tasks::blueprint_load::LoadedTargetBlueprint; use super::tasks::blueprint_planner; use super::tasks::blueprint_pruner; use super::tasks::blueprint_rendezvous; +use super::tasks::capacity_metrics; use super::tasks::crdb_node_id_collector; use super::tasks::decommissioned_disk_cleaner; use super::tasks::dns_config; @@ -250,6 +251,7 @@ impl BackgroundTasksInitializer { task_region_replacement: Activator::new(), task_region_replacement_driver: Activator::new(), task_instance_watcher: Activator::new(), + task_capacity_metrics: Activator::new(), task_instance_updater: Activator::new(), task_instance_reincarnation: Activator::new(), task_service_firewall_propagation: Activator::new(), @@ -350,6 +352,7 @@ impl BackgroundTasksInitializer { task_region_replacement_driver, task_instance_watcher, task_instance_updater, + task_capacity_metrics, task_instance_reincarnation, task_service_firewall_propagation, task_abandoned_vmm_reaper, @@ -808,6 +811,23 @@ impl BackgroundTasksInitializer { }) }; + { + let capacity_metrics = capacity_metrics::CapacityMetrics::new( + datastore.clone(), + producer_registry, + nexus_id, + ); + driver.register(TaskDefinition { + name: "capacity_metrics", + description: "periodically emits quota capacity metrics", + period: config.capacity_metrics.period_secs, + task_impl: Box::new(capacity_metrics), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_capacity_metrics, + }); + } + // Background task: schedule update sagas for instances in need of // state updates. { diff --git a/nexus/src/app/background/tasks/capacity_metrics.rs b/nexus/src/app/background/tasks/capacity_metrics.rs new file mode 100644 index 00000000000..51fc070f858 --- /dev/null +++ b/nexus/src/app/background/tasks/capacity_metrics.rs @@ -0,0 +1,388 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task that collects and emits silo quota metrics: +//! +//! * silo_quota:{memory,cpu,virtual_storage}_capacity: resource limits set on +//! the silo. +//! * silo_quota:{memory,cpu,virtual_storage}_provisioned: resources charged +//! against the silo. +//! +//! Note: because background tasks run on each Nexus instance, these metrics are +//! emitted once for each Nexus--as of this writing, in triplicate. Callers +//! should take the mean or maximum datum value over Nexus instance IDs to +//! deduplicate. +//! +//! Note: the `silo_quota` metrics are related to, but distinct from, the +//! `collection_target` metric. The `silo_quota` metrics are emitted on a regular +//! interval, and labeled by silo ID. Meanwhile, `collection_target` is currently +//! emitted only when silo utilization changes, resulting in a sparse timeseries, +//! and only covers utilization. Users who wish to show quota utilization ratios +//! (i.e. utilization divided by capacity), or who require regular timeseries, +//! should use the `silo_quota` metrics. + +use crate::app::background::BackgroundTask; +use dropshot::PaginationOrder; +use futures::FutureExt; +use futures::future::BoxFuture; +use nexus_auth::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_db_queries::db::datastore::SQL_BATCH_SIZE; +use nexus_db_queries::db::pagination::Paginator; +use nexus_types::internal_api::background::CapacityMetricsStatus; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::OmicronZoneUuid; +use oximeter::MetricsError; +use oximeter::Sample; +use oximeter::queue::BoundedQueue; +use oximeter::types::ProducerRegistry; +use std::sync::{Arc, Mutex}; + +oximeter::use_timeseries!("silo-quota.toml"); + +use silo_quota::SiloQuota; + +const QUEUE_SIZE: usize = 8192; + +pub(crate) struct CapacityMetrics { + datastore: Arc, + metrics: Arc>>, + nexus_id: OmicronZoneUuid, +} + +impl CapacityMetrics { + pub(crate) fn new( + datastore: Arc, + producer_registry: &ProducerRegistry, + nexus_id: OmicronZoneUuid, + ) -> Self { + let metrics = Arc::new(Mutex::new(BoundedQueue::new(QUEUE_SIZE))); + producer_registry.register_producer(Producer(metrics.clone())).unwrap(); + Self { datastore, metrics, nexus_id } + } + + async fn fetch_samples( + &self, + opctx: &OpContext, + nexus_id: OmicronZoneUuid, + ) -> anyhow::Result { + let now = chrono::Utc::now(); + let nexus_id = nexus_id.into_untyped_uuid(); + + let mut paginator = + Paginator::new(SQL_BATCH_SIZE, PaginationOrder::Ascending); + let mut samples = Vec::::new(); + + while let Some(p) = paginator.next() { + let silo_utils = self + .datastore + .silo_utilization_list( + opctx, + &PaginatedBy::Id(p.current_pagparams()), + ) + .await?; + paginator = p.found_batch(&silo_utils, &|util| util.silo_id); + + for util in silo_utils { + let target = SiloQuota { silo_id: util.silo_id, nexus_id }; + samples.extend([ + Sample::new_with_timestamp( + now, + &target, + &silo_quota::MemoryCapacity { + datum: util.memory_allocated.to_bytes(), + }, + )?, + Sample::new_with_timestamp( + now, + &target, + &silo_quota::MemoryProvisioned { + datum: util.memory_provisioned.to_bytes(), + }, + )?, + Sample::new_with_timestamp( + now, + &target, + &silo_quota::CpuCapacity { + datum: u64::try_from(util.cpus_allocated)?, + }, + )?, + Sample::new_with_timestamp( + now, + &target, + &silo_quota::CpuProvisioned { + datum: u64::try_from(util.cpus_provisioned)?, + }, + )?, + Sample::new_with_timestamp( + now, + &target, + &silo_quota::VirtualStorageCapacity { + datum: util.storage_allocated.to_bytes(), + }, + )?, + Sample::new_with_timestamp( + now, + &target, + &silo_quota::VirtualStorageProvisioned { + datum: util.storage_provisioned.to_bytes(), + }, + )?, + ]); + } + } + + let mut metrics = self.metrics.lock().unwrap(); + let n_new_samples = samples.len(); + let n_current_samples = metrics.len(); + let n_dropped_samples = metrics.extend(samples); + if n_dropped_samples > 0 { + slog::warn!( + opctx.log, + "sample queue is too full, dropping oldest samples"; + "n_new_samples" => n_new_samples, + "n_current_samples" => n_current_samples, + "n_dropped_samples" => n_dropped_samples, + ); + } + + Ok(n_new_samples.min(QUEUE_SIZE)) + } +} + +impl BackgroundTask for CapacityMetrics { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + async move { + match self.fetch_samples(opctx, self.nexus_id).await { + Ok(sample_count) => { + serde_json::json!(CapacityMetricsStatus { + samples_queued: sample_count, + }) + } + Err(error) => { + serde_json::json!({"error": format!("{error:#}")}) + } + } + } + .boxed() + } +} + +#[derive(Debug)] +struct Producer(Arc>>); + +impl oximeter::Producer for Producer { + fn produce( + &mut self, + ) -> Result>, MetricsError> { + let samples = self.0.lock().unwrap().drain(); + Ok(Box::new(samples.into_iter())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nexus_auth::{authn, authz}; + use nexus_db_model::DnsGroup; + use nexus_db_model::Project; + use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder; + use nexus_test_utils::http_testing::AuthnMode; + use nexus_test_utils::resource_helpers::{ + create_local_user, grant_iam, test_params, + }; + use nexus_test_utils_macros::nexus_test; + use nexus_types::external_api::policy::SiloRole; + use nexus_types::external_api::project::ProjectCreate; + use nexus_types::external_api::silo::{ + SiloCreate, SiloIdentityMode, SiloQuotasCreate, + }; + use nexus_types::identity::Resource; + use omicron_common::api::external::{ + ByteCount, IdentityMetadataCreateParams, + }; + use omicron_uuid_kinds::InstanceUuid; + use oximeter::types::{Datum, FieldValue, ProducerResultsItem}; + use std::collections::BTreeMap; + use uuid::Uuid; + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + fn collect_samples(registry: &ProducerRegistry) -> Vec { + let mut results = registry.collect(); + assert_eq!(results.len(), 1); + match results.pop().unwrap() { + ProducerResultsItem::Ok(samples) => samples, + ProducerResultsItem::Err(error) => { + panic!("produce failed: {error}") + } + } + } + + // Integration test for the CapacityMetrics task. Create silos, projects, + // and utilization records; collect samples from the producer registry; + // and assert the expected samples. + #[nexus_test(server = crate::Server)] + async fn test_capacity_metrics(cptestctx: &ControlPlaneTestContext) { + let datastore = cptestctx.server.server_context().nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + + let mut silos = Vec::new(); + for (name, cpus, memory, storage) in + [("populated", 32, 128, 1024), ("empty", 8, 32, 256)] + { + let silo = datastore + .silo_create( + &opctx, + &opctx, + SiloCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "Capacity metrics fixture".into(), + }, + quotas: SiloQuotasCreate { + cpus, + memory: ByteCount::from_gibibytes_u32(memory), + storage: ByteCount::from_gibibytes_u32(storage), + }, + identity_mode: SiloIdentityMode::LocalOnly, + admin_group_name: None, + tls_certificates: vec![], + mapped_fleet_roles: Default::default(), + }, + &[], + DnsVersionUpdateBuilder::new( + DnsGroup::External, + "capacity metrics fixture".into(), + "test_capacity_metrics".into(), + ), + ) + .await + .unwrap(); + silos.push(silo); + } + let populated_silo = silos[0].id(); + let empty_silo = silos[1].id(); + let client = &cptestctx.external_client; + let admin_user = create_local_user( + client, + &silos[0].clone().try_into().unwrap(), + &"admin".parse().unwrap(), + test_params::UserPassword::LoginDisallowed, + ) + .await; + grant_iam( + client, + &format!("/v1/system/silos/{populated_silo}"), + SiloRole::Admin, + admin_user.id, + AuthnMode::PrivilegedUser, + ) + .await; + let setup_opctx = OpContext::for_background( + opctx.log.clone(), + Arc::new(authz::Authz::new(&opctx.log)), + authn::Context::for_test_user( + admin_user.id, + populated_silo, + authn::SiloAuthnPolicy::try_from(&silos[0]).unwrap(), + ), + datastore.clone(), + ); + let project = Project::new( + populated_silo, + ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "capacity-metrics-project".parse().unwrap(), + description: "Capacity metrics fixture".into(), + }, + defaults: None, + }, + ); + let (_, project) = + datastore.project_create(&setup_opctx, project).await.unwrap(); + let project_id = project.id(); + + datastore + .virtual_provisioning_collection_insert_instance( + &opctx, + InstanceUuid::new_v4(), + project_id, + 4, + ByteCount::from_gibibytes_u32(16).into(), + ) + .await + .unwrap(); + + let registry = ProducerRegistry::new(); + let nexus_id = OmicronZoneUuid::new_v4(); + let mut task = + CapacityMetrics::new(datastore.clone(), ®istry, nexus_id); + assert!(collect_samples(®istry).is_empty()); + + datastore + .virtual_provisioning_collection_insert_disk( + &opctx, + Uuid::new_v4(), + project_id, + ByteCount::from_gibibytes_u32(100).into(), + ) + .await + .unwrap(); + + assert_eq!( + task.activate(&opctx).await, + serde_json::json!({"samples_queued": 12}), + ); + let samples = collect_samples(®istry); + assert_eq!(samples.len(), 12); + + let mut expected = BTreeMap::new(); + for (silo_id, kind, cpu, memory, storage) in [ + (populated_silo, "capacity", 32, 128, 1024), + (populated_silo, "provisioned", 4, 16, 100), + (empty_silo, "capacity", 8, 32, 256), + (empty_silo, "provisioned", 0, 0, 0), + ] { + for (metric, value) in [ + ("cpu", cpu), + ("memory", ByteCount::from_gibibytes_u32(memory).to_bytes()), + ( + "virtual_storage", + ByteCount::from_gibibytes_u32(storage).to_bytes(), + ), + ] { + expected.insert( + (silo_id, format!("silo_quota:{metric}_{kind}")), + value, + ); + } + } + + for sample in samples { + let fields = sample.sorted_target_fields(); + let FieldValue::Uuid(silo_id) = fields["silo_id"].value else { + panic!("expected silo_id be a uuid"); + }; + let key = (silo_id, sample.timeseries_name.to_string()); + let value = expected.remove(&key).unwrap_or_else(|| { + panic!("unexpected or duplicate sample: {key:?}") + }); + assert_eq!(sample.measurement.datum(), &Datum::U64(value)); + } + assert!(expected.is_empty(), "missing samples: {expected:?}"); + assert!( + collect_samples(®istry).is_empty(), + "produce must drain the buffer" + ); + } +} diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs index aabf38770a3..2281c27f8e6 100644 --- a/nexus/src/app/background/tasks/mod.rs +++ b/nexus/src/app/background/tasks/mod.rs @@ -14,6 +14,7 @@ pub mod blueprint_load; pub mod blueprint_planner; pub mod blueprint_pruner; pub mod blueprint_rendezvous; +pub mod capacity_metrics; pub mod crdb_node_id_collector; pub mod decommissioned_disk_cleaner; pub mod dns_config; diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index abea37101e2..c75cf708c49 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -148,6 +148,7 @@ region_replacement.period_secs = 999999 # signal" instead of "triggered by a periodic timer firing" region_replacement_driver.period_secs = 999999 instance_watcher.period_secs = 30 +capacity_metrics.period_secs = 60 service_firewall_propagation.period_secs = 300 v2p_mapping_propagation.period_secs = 30 abandoned_vmm_reaper.period_secs = 60 diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 19cbf05bb01..8f81da50645 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -36,6 +36,13 @@ use swrite::swriteln; use tufaceous_artifact::ArtifactHash; use uuid::Uuid; +/// The status of a successful `capacity_metrics` background task activation. +#[derive(Debug, Serialize, Deserialize)] +pub struct CapacityMetricsStatus { + /// Number of new samples retained in the producer queue. + pub samples_queued: usize, +} + /// The status of a `region_replacement` background task activation #[derive(Debug, Serialize, Deserialize, Default)] pub struct RegionReplacementStatus { diff --git a/oximeter/oximeter/schema/silo-quota.toml b/oximeter/oximeter/schema/silo-quota.toml new file mode 100644 index 00000000000..83fa6730337 --- /dev/null +++ b/oximeter/oximeter/schema/silo-quota.toml @@ -0,0 +1,71 @@ +format_version = 1 + +[target] +name = "silo_quota" +description = "Silo quota capacity and provisioned resources" +authz_scope = "fleet" +versions = [ + { version = 1, fields = [ "nexus_id", "silo_id" ] }, +] + +[fields.nexus_id] +type = "uuid" +description = "ID of the Nexus reporting the sample. We emit quota metrics from each Nexus, so aggregate over Nexus instances by mean or max, but not sum, to avoid double-counting." + +[fields.silo_id] +type = "uuid" +description = "ID of the silo" + +[[metrics]] +name = "memory_capacity" +description = "Quota capacity for provisioned memory" +units = "bytes" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "memory_provisioned" +description = "Provisioned memory charged against the quota" +units = "bytes" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "cpu_capacity" +description = "Quota capacity for provisioned vCPUs" +units = "count" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "cpu_provisioned" +description = "Provisioned vCPUs charged against the quota" +units = "count" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "virtual_storage_capacity" +description = "Quota capacity for provisioned virtual disk and snapshot storage" +units = "bytes" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "virtual_storage_provisioned" +description = "Provisioned virtual disk and snapshot storage charged against the quota" +units = "bytes" +datum_type = "u64" +versions = [ + { added_in = 1, fields = [] } +] diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index ebc221cf69a..a1021d7a46e 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -66,6 +66,7 @@ region_replacement_driver.period_secs = 10 service_firewall_propagation.period_secs = 300 v2p_mapping_propagation.period_secs = 30 instance_watcher.period_secs = 30 +capacity_metrics.period_secs = 60 abandoned_vmm_reaper.period_secs = 60 saga_recovery.period_secs = 600 lookup_region_port.period_secs = 60 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index 5769ae89f23..a0264ef74e5 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -66,6 +66,7 @@ region_replacement_driver.period_secs = 10 service_firewall_propagation.period_secs = 300 v2p_mapping_propagation.period_secs = 30 instance_watcher.period_secs = 30 +capacity_metrics.period_secs = 60 abandoned_vmm_reaper.period_secs = 60 saga_recovery.period_secs = 600 lookup_region_port.period_secs = 60