From 3076a3d3745bc225c1b209738dc28872e21fecb1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 5 Nov 2026 12:34:58 -0800 Subject: [PATCH 1/4] [fm] add in-service sleds to fault management analysis input --- nexus/db-queries/src/db/datastore/fm.rs | 2 ++ nexus/fm/preparation/src/lib.rs | 30 ++++++++++++++-- nexus/fm/src/analysis_input.rs | 30 +++++++++++++--- nexus/fm/src/diagnosis/power_shelf.rs | 1 + nexus/fm/src/test_util.rs | 3 ++ .../output/analysis_input_report_empty.out | 2 ++ .../output/analysis_input_report_same_inv.out | 2 ++ .../analysis_input_report_with_cases.out | 3 ++ nexus/types/src/fm/analysis_reports.rs | 36 ++++++++++++++++++- 9 files changed, 102 insertions(+), 7 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/fm.rs b/nexus/db-queries/src/db/datastore/fm.rs index e49d8b5ac46..26163bc54c3 100644 --- a/nexus/db-queries/src/db/datastore/fm.rs +++ b/nexus/db-queries/src/db/datastore/fm.rs @@ -3193,6 +3193,7 @@ mod tests { open_cases: Default::default(), closed_cases_copied_forward: Default::default(), in_service_disks: Default::default(), + in_service_sleds: Default::default(), observed_sagas: Default::default(), observed_silo_certificates: Default::default(), config: None, @@ -3836,6 +3837,7 @@ mod tests { num_ereporter_restarts: 0, unknown_restart_ereport_ids: Default::default(), in_service_disks: Default::default(), + in_service_sleds: Default::default(), observed_sagas: Default::default(), observed_silo_certificates: Default::default(), config: None, diff --git a/nexus/fm/preparation/src/lib.rs b/nexus/fm/preparation/src/lib.rs index 9016bf23381..53937f3360a 100644 --- a/nexus/fm/preparation/src/lib.rs +++ b/nexus/fm/preparation/src/lib.rs @@ -23,6 +23,7 @@ use nexus_db_queries::db::model::SagaReasonAbandoned; use nexus_db_queries::db::model::ServiceKind; use nexus_db_queries::db::pagination::Paginator; use nexus_fm as fm; +use nexus_types::deployment::SledFilter; use nexus_types::in_service_disk::InServiceDisk; use nexus_types::internal_api::background::fm_analysis as status; use nexus_types::observed_certificate::ObservedCertificate; @@ -36,9 +37,11 @@ use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_uuid_kinds::AlertUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SupportBundleUuid; use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; @@ -123,7 +126,7 @@ impl InputLoader { // Spawn each preparation step in parallel. // // N.B. that right now, the concurrency limit here is just the number of - // steps that are defined. Right now, there are only 7 preparation + // steps that are defined. Right now, there are only 8 preparation // steps, so that's fine. In the future, if there are a whole lot of // steps, we may eventually want to think about having a concurrency // limit here, especially if steps can recursively spawn other @@ -171,6 +174,9 @@ impl InputLoader { Ok(LoadedInput::InServiceDisks(disks)) => { builder = builder.in_service_disks(Arc::new(disks)); } + Ok(LoadedInput::InServiceSleds(sleds)) => { + builder = builder.in_service_sleds(Arc::new(sleds)); + } Ok(LoadedInput::ObservedSagas(sagas)) => { builder = builder.observed_sagas(Arc::new(sagas)); } @@ -255,6 +261,10 @@ impl InputLoader { .load_in_service_disks(opctx, &mut warnings) .await .map(LoadedInput::InServiceDisks), + InputStep::InServiceSleds => self + .load_in_service_sleds(opctx) + .await + .map(LoadedInput::InServiceSleds), InputStep::ObservedSagas => self .prepare_observed_sagas(opctx) .await @@ -353,6 +363,22 @@ impl InputLoader { Ok(in_service_disks) } + /// Load the IDs of all in-service sleds. + /// + /// Like [`Self::load_in_service_disks`], this reads the executed DB view + /// rather than the target blueprint. + async fn load_in_service_sleds( + &self, + opctx: &OpContext, + ) -> anyhow::Result> { + let sleds = self + .datastore + .sled_list_all_batched(opctx, SledFilter::InService) + .await + .context("failed to load in-service sleds")?; + Ok(sleds.into_iter().map(|sled| sled.id()).collect()) + } + /// Build the certificate diagnosis engine's input: every silo, with the /// leaf validity window of each of its external TLS certificates. /// @@ -698,6 +724,7 @@ struct StepOutput { #[strum_discriminants(strum(serialize_all = "snake_case"))] enum LoadedInput { InServiceDisks(IdOrdMap), + InServiceSleds(BTreeSet), ObservedSagas(IdOrdMap), SiloCertificates(IdOrdMap), EreporterRestarts(Vec), @@ -732,7 +759,6 @@ mod tests { use omicron_uuid_kinds::CaseUuid; use omicron_uuid_kinds::SitrepUuid; use serde_json::json; - use std::collections::BTreeSet; /// Exercises the wiring in `load_existing_alert_markers`: alert request /// ids are collected from the parent sitrep's *closed* cases only, their diff --git a/nexus/fm/src/analysis_input.rs b/nexus/fm/src/analysis_input.rs index 9bd0b7ea5f3..7698c71d0ec 100644 --- a/nexus/fm/src/analysis_input.rs +++ b/nexus/fm/src/analysis_input.rs @@ -14,6 +14,7 @@ use nexus_types::inventory; use nexus_types::observed_certificate::ObservedSiloCertificates; use nexus_types::observed_saga::ObservedSaga; use omicron_uuid_kinds::AlertUuid; +use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::SupportBundleUuid; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -62,6 +63,7 @@ pub struct Input { /// new sitrep's `support_bundle_generation`. support_bundles_changed: bool, in_service_disks: Arc>, + in_service_sleds: Arc>, /// All non-terminal (running, unwinding, or abandoned) sagas, annotated /// with their latest node-event time and owning-Nexus state. observed_sagas: Arc>, @@ -123,6 +125,11 @@ impl Input { &self.in_service_disks } + /// All in-service sleds (`SledFilter::InService` in the DB). + pub fn in_service_sleds(&self) -> &BTreeSet { + &self.in_service_sleds + } + /// All non-terminal sagas observed in the database, indexed by `saga_id`. /// See the saga diagnosis engine for how absence (a saga that has reached /// a terminal state) drives case closure. @@ -151,10 +158,10 @@ impl Input { /// Returns a [`Builder`] for constructing a new `Input` from the provided /// `parent_sitrep` and inventory collection. /// - /// The queried input collections (in-service disks, observed sagas, - /// observed silo certificates) and the FM configuration are provided via - /// the builder's setters; [`Builder::build`] fails if any was never - /// provided. + /// The queried input collections (in-service disks and sleds, observed + /// sagas, observed silo certificates) and the FM configuration are + /// provided via the builder's setters; [`Builder::build`] fails if any was + /// never provided. /// /// # Errors /// @@ -189,6 +196,7 @@ impl Input { parent_sitrep, inv, in_service_disks: None, + in_service_sleds: None, observed_sagas: None, observed_silo_certificates: None, config: None, @@ -217,6 +225,7 @@ pub struct Builder { // run analysis against an empty collection (which has active semantics, // e.g. an empty `observed_sagas` reads as "every saga case may close"). in_service_disks: Option>>, + in_service_sleds: Option>>, observed_sagas: Option>>, observed_silo_certificates: Option>>, /// The FM configuration. Required for the same reason as the queried @@ -354,6 +363,13 @@ impl Builder { self } + /// Provides the in-service sleds queried from the database. + /// Required; [`Builder::build`] fails without it. + pub fn in_service_sleds(mut self, sleds: Arc>) -> Self { + self.in_service_sleds = Some(sleds); + self + } + /// Provides the non-terminal sagas queried from the database. /// Required; [`Builder::build`] fails without it. pub fn observed_sagas( @@ -390,6 +406,7 @@ impl Builder { #[cfg(any(test, feature = "testing"))] pub fn with_empty_defaults(mut self) -> Self { self.in_service_disks.get_or_insert_with(Default::default); + self.in_service_sleds.get_or_insert_with(Default::default); self.observed_sagas.get_or_insert_with(Default::default); self.observed_silo_certificates.get_or_insert_with(Default::default); self.config.get_or_insert_with(Default::default); @@ -400,6 +417,9 @@ impl Builder { let in_service_disks = self .in_service_disks .ok_or(MissingInput { name: "in_service_disks" })?; + let in_service_sleds = self + .in_service_sleds + .ok_or(MissingInput { name: "in_service_sleds" })?; let observed_sagas = self .observed_sagas .ok_or(MissingInput { name: "observed_sagas" })?; @@ -443,6 +463,7 @@ impl Builder { .iter() .map(|d| d.physical_disk_id) .collect(), + in_service_sleds: (*in_service_sleds).clone(), observed_sagas: observed_sagas .iter() .map(|s| { @@ -580,6 +601,7 @@ impl Builder { alerts_changed, support_bundles_changed, in_service_disks, + in_service_sleds, observed_sagas, observed_silo_certificates, config, diff --git a/nexus/fm/src/diagnosis/power_shelf.rs b/nexus/fm/src/diagnosis/power_shelf.rs index 2b8822ab1db..381d190212c 100644 --- a/nexus/fm/src/diagnosis/power_shelf.rs +++ b/nexus/fm/src/diagnosis/power_shelf.rs @@ -1003,6 +1003,7 @@ mod tests { parent, collection.into(), Arc::new(IdOrdMap::new()), + Arc::new(BTreeSet::new()), Arc::new(IdOrdMap::new()), Arc::new(IdOrdMap::new()), Default::default(), diff --git a/nexus/fm/src/test_util.rs b/nexus/fm/src/test_util.rs index cb6b7d5117f..7380039351c 100644 --- a/nexus/fm/src/test_util.rs +++ b/nexus/fm/src/test_util.rs @@ -39,6 +39,7 @@ use omicron_uuid_kinds::SitrepUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use rand::rngs::StdRng; +use std::collections::BTreeSet; use std::sync::Arc; use std::sync::Mutex; use typed_rng::TypedUuidRng; @@ -86,12 +87,14 @@ impl FmTest { parent_sitrep: Option>, inv: Arc, in_service_disks: Arc>, + in_service_sleds: Arc>, observed_sagas: Arc>, observed_silo_certificates: Arc>, config: FmConfig, ) -> Result { let mut builder = Input::builder(parent_sitrep, inv)? .in_service_disks(in_service_disks) + .in_service_sleds(in_service_sleds) .observed_sagas(observed_sagas) .observed_silo_certificates(observed_silo_certificates) .config(config); diff --git a/nexus/types/output/analysis_input_report_empty.out b/nexus/types/output/analysis_input_report_empty.out index 09975971526..5f953b048e9 100644 --- a/nexus/types/output/analysis_input_report_empty.out +++ b/nexus/types/output/analysis_input_report_empty.out @@ -8,6 +8,8 @@ no cases copied forward no in-service control plane disks +no in-service sleds + no non-terminal sagas observed no silos observed diff --git a/nexus/types/output/analysis_input_report_same_inv.out b/nexus/types/output/analysis_input_report_same_inv.out index 4b5d6ebbdb5..fd9608e38ac 100644 --- a/nexus/types/output/analysis_input_report_same_inv.out +++ b/nexus/types/output/analysis_input_report_same_inv.out @@ -10,6 +10,8 @@ no cases copied forward no in-service control plane disks +no in-service sleds + no non-terminal sagas observed no silos observed diff --git a/nexus/types/output/analysis_input_report_with_cases.out b/nexus/types/output/analysis_input_report_with_cases.out index 16f8052fb06..0385dfef8eb 100644 --- a/nexus/types/output/analysis_input_report_with_cases.out +++ b/nexus/types/output/analysis_input_report_with_cases.out @@ -42,6 +42,9 @@ in-service control plane disks (2 total): * disk 11111111-1111-1111-1111-111111111111 * disk 22222222-2222-2222-2222-222222222222 +in-service sleds (1 total): + * sled 33333333-3333-3333-3333-333333333333 + non-terminal sagas observed (2 total): * saga 5a9a0001-5a9a-45a9-85a9-5a9a5a9a5a9a (fake-saga): state: Unwinding diff --git a/nexus/types/src/fm/analysis_reports.rs b/nexus/types/src/fm/analysis_reports.rs index 68b4736d08d..3a6bbcd6b90 100644 --- a/nexus/types/src/fm/analysis_reports.rs +++ b/nexus/types/src/fm/analysis_reports.rs @@ -15,7 +15,7 @@ use iddqd::IdOrdMap; use omicron_uuid_kinds::SiloUuid; use omicron_uuid_kinds::{ AlertUuid, CaseUuid, CollectionUuid, PhysicalDiskUuid, SitrepUuid, - SupportBundleUuid, + SledUuid, SupportBundleUuid, }; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -537,6 +537,10 @@ pub struct InputReport { /// engines for this analysis pass. #[serde(default)] pub in_service_disks: BTreeSet, + /// All in-service sleds visible to the diagnosis engines for this + /// analysis pass. + #[serde(default)] + pub in_service_sleds: BTreeSet, /// All non-terminal sagas visible to the diagnosis engines for this /// analysis pass. #[serde(default)] @@ -667,6 +671,7 @@ impl fmt::Display for InputReportMultilineDisplay<'_> { num_ereporter_restarts, unknown_restart_ereport_ids, in_service_disks, + in_service_sleds, observed_sagas, observed_silo_certificates, config, @@ -985,6 +990,27 @@ impl fmt::Display for InputReportMultilineDisplay<'_> { } } + if in_service_sleds.is_empty() { + writeln!( + f, + "\n{:indent$}{}", + "", + styles.heading().style("no in-service sleds"), + )?; + } else { + writeln!( + f, + "\n{:indent$}{} ({} total):", + "", + heading.style("in-service sleds"), + in_service_sleds.len() + )?; + let indent = indent + 2; + for sled_id in in_service_sleds { + writeln!(f, "{:indent$}* sled {sled_id}", "")?; + } + } + if observed_sagas.is_empty() { writeln!(f, "\n{:indent$}no non-terminal sagas observed", "")?; } else { @@ -1186,6 +1212,11 @@ mod tests { .unwrap(), ); + let in_service_sleds = BTreeSet::from([SledUuid::from_str( + "33333333-3333-3333-3333-333333333333", + ) + .unwrap()]); + let observed_sagas = example_observed_sagas(); let observed_silo_certificates = example_observed_silo_certificates(); @@ -1206,6 +1237,7 @@ mod tests { open_cases, closed_cases_copied_forward, in_service_disks, + in_service_sleds, observed_sagas, observed_silo_certificates, config: Some(ConfigReport { @@ -1283,6 +1315,7 @@ mod tests { open_cases: BTreeMap::new(), closed_cases_copied_forward: BTreeMap::new(), in_service_disks: BTreeSet::new(), + in_service_sleds: BTreeSet::new(), observed_sagas: BTreeMap::new(), observed_silo_certificates: BTreeMap::new(), config: None, @@ -1308,6 +1341,7 @@ mod tests { open_cases: BTreeMap::new(), closed_cases_copied_forward: BTreeMap::new(), in_service_disks: BTreeSet::new(), + in_service_sleds: BTreeSet::new(), observed_sagas: BTreeMap::new(), observed_silo_certificates: BTreeMap::new(), config: Some(ConfigReport { From 10afa15af13a2b371b1018b169ace8d95f93ffe1 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 5 Nov 2026 12:41:30 -0800 Subject: [PATCH 2/4] move is_propolis_zone into nexus-types for reuse by fault management --- nexus/src/app/update.rs | 17 ++--------------- nexus/types/src/inventory.rs | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/nexus/src/app/update.rs b/nexus/src/app/update.rs index b6f2bfa7176..ee4eca6df91 100644 --- a/nexus/src/app/update.rs +++ b/nexus/src/app/update.rs @@ -13,7 +13,6 @@ use chrono::Utc; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; use futures::Stream; -use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use nexus_auth::authz; use nexus_db_lookup::LookupPath; use nexus_db_model::TufRepoUpload; @@ -31,6 +30,7 @@ use nexus_types::identity::Asset; use nexus_types::internal_api::views as internal_views; use nexus_types::inventory::Collection; use nexus_types::inventory::Zpool; +use nexus_types::inventory::is_propolis_zone; use nexus_types::tuf_repo::TufRepoDescription; use omicron_common::api::external::InternalContext; use omicron_common::api::external::Nullable; @@ -219,20 +219,6 @@ impl UpdateContactSupportChecksInput { } } -// String matching here because it is the most straightforward way to check -// if a zone is a propolis zone or not. This way of identifying a zone is not -// ideal but in this case it makes sense. This `contact_support` field will only -// be in use until a proper FM implementation is in place. The only consequence -// of this pattern matching failing (e.g. the propolis zone prefix changes and -// is also no longer set by this constant) would be that the contact_support -// field reports a false positive, but would have no further impact than that. -// -// This function should not be used for anything other than filtering propolis -// zones out of the `enabled_smf_services_not_online_by_sled` field -fn is_propolis_zone(zone: &str) -> bool { - zone.starts_with(PROPOLIS_ZONE_PREFIX) -} - /// Identifies a saga that has been running or unwinding for too long. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] struct StuckSaga { @@ -902,6 +888,7 @@ async fn component_version_counts( mod test { use super::*; use chrono::Utc; + use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use nexus_db_model::saga_types::Saga; use nexus_db_model::saga_types::SecId; use nexus_inventory::CollectionBuilder; diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 0bdda5631e7..20334784fb3 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -22,6 +22,7 @@ pub use gateway_types::rot::RotSlot; use iddqd::IdOrdItem; use iddqd::IdOrdMap; use iddqd::id_upcast; +use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use omicron_common::address::Ipv6Subnet; use omicron_common::address::{RACK_PREFIX_LENGTH, SLED_PREFIX_LENGTH}; use omicron_common::api::external::ByteCount; @@ -210,6 +211,19 @@ pub struct Collection { pub internal_dns_generation_status: IdOrdMap, } +/// Returns whether `zone_name` names a Propolis zone. +/// +/// Propolis zones come and go with their VMMs, so SMF services in them are +/// excluded when judging sled health from +/// [`SledAgent::smf_services_enabled_not_online`]. +/// +/// This matches on the zone name prefix. If that prefix ever changed without +/// [`PROPOLIS_ZONE_PREFIX`] following, Propolis services would be reported as +/// false positives. +pub fn is_propolis_zone(zone_name: &str) -> bool { + zone_name.starts_with(PROPOLIS_ZONE_PREFIX) +} + impl Collection { pub fn host_phase_1_active_slot_for( &self, From d352afe84c7980c90319a52d6b8c98a6d63304e8 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 5 Nov 2026 12:52:16 -0800 Subject: [PATCH 3/4] [fm] add SMF service diagnosis engine for services in maintenance --- Cargo.lock | 2 + dev-tools/omdb/tests/successes.out | 6 + nexus/db-model/src/fm.rs | 2 + nexus/db-model/src/fm/diagnosis_engine.rs | 3 + nexus/db-model/src/fm/fact_smf_service.rs | 117 ++++ nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-queries/src/db/datastore/fm.rs | 97 ++- nexus/db-schema/src/enums.rs | 1 + nexus/db-schema/src/schema.rs | 18 + nexus/fm/Cargo.toml | 2 + nexus/fm/src/diagnosis/mod.rs | 2 + nexus/fm/src/diagnosis/smf_service.rs | 732 ++++++++++++++++++++++ nexus/fm/src/test_util.rs | 1 + nexus/types/src/fm.rs | 4 +- nexus/types/src/fm/case.rs | 8 +- nexus/types/src/fm/fact.rs | 57 +- schema/crdb/dbinit.sql | 44 +- schema/crdb/fm-smf-service-de/up1.sql | 1 + schema/crdb/fm-smf-service-de/up2.sql | 3 + schema/crdb/fm-smf-service-de/up3.sql | 31 + 20 files changed, 1127 insertions(+), 7 deletions(-) create mode 100644 nexus/db-model/src/fm/fact_smf_service.rs create mode 100644 nexus/fm/src/diagnosis/smf_service.rs create mode 100644 schema/crdb/fm-smf-service-de/up1.sql create mode 100644 schema/crdb/fm-smf-service-de/up2.sql create mode 100644 schema/crdb/fm-smf-service-de/up3.sql diff --git a/Cargo.lock b/Cargo.lock index 8410a7c2bd5..193f4fc54e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7696,6 +7696,7 @@ dependencies = [ "chrono", "ereport-types", "iddqd", + "illumos-utils", "nexus-db-model", "nexus-fm-slippy", "nexus-inventory", @@ -7711,6 +7712,7 @@ dependencies = [ "rand 0.9.2", "serde", "serde_json", + "sled-agent-types", "slog", "slog-error-chain 0.1.0 (git+https://github.com/oxidecomputer/slog-error-chain?branch=main)", "steno", diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 0e1ddc21071..1ee705b1c83 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -841,6 +841,7 @@ task: "fm_analysis" ereporter_restarts: ms ereports: ms in_service_disks: ms + in_service_sleds: ms observed_sagas: ms silo_certificates: ms support_bundle_markers: ms @@ -917,6 +918,8 @@ task: "fm_sitrep_gc" batches: 1 orphaned fm_fact_saga rows deleted: 0 batches: 1 + orphaned fm_fact_smf_service rows deleted: 0 + batches: 1 orphaned fm_sitrep_analysis_report rows deleted: batches: 1 orphaned fm_support_bundle_request rows deleted: 0 @@ -1604,6 +1607,7 @@ task: "fm_analysis" ereporter_restarts: ms ereports: ms in_service_disks: ms + in_service_sleds: ms observed_sagas: ms silo_certificates: ms support_bundle_markers: ms @@ -1680,6 +1684,8 @@ task: "fm_sitrep_gc" batches: 1 orphaned fm_fact_saga rows deleted: 0 batches: 1 + orphaned fm_fact_smf_service rows deleted: 0 + batches: 1 orphaned fm_sitrep_analysis_report rows deleted: batches: 1 orphaned fm_support_bundle_request rows deleted: 0 diff --git a/nexus/db-model/src/fm.rs b/nexus/db-model/src/fm.rs index cad506ecddc..fc0c8879f62 100644 --- a/nexus/db-model/src/fm.rs +++ b/nexus/db-model/src/fm.rs @@ -38,6 +38,8 @@ mod fact_physical_disk; pub use fact_physical_disk::*; mod fact_saga; pub use fact_saga::*; +mod fact_smf_service; +pub use fact_smf_service::*; mod rendezvous_created; pub use rendezvous_created::*; mod support_bundle_request; diff --git a/nexus/db-model/src/fm/diagnosis_engine.rs b/nexus/db-model/src/fm/diagnosis_engine.rs index fb1d4393d83..4dd70e0d1fe 100644 --- a/nexus/db-model/src/fm/diagnosis_engine.rs +++ b/nexus/db-model/src/fm/diagnosis_engine.rs @@ -27,6 +27,7 @@ impl_enum_type!( PhysicalDisk => b"physical_disk" Saga => b"saga" Certificate => b"certificate" + SmfService => b"smf_service" ); @@ -41,6 +42,7 @@ impl From for fm::DiagnosisEngineKind { DiagnosisEngine::Certificate => { fm::DiagnosisEngineKind::Certificate } + DiagnosisEngine::SmfService => fm::DiagnosisEngineKind::SmfService, } } } @@ -56,6 +58,7 @@ impl From for DiagnosisEngine { fm::DiagnosisEngineKind::Certificate => { DiagnosisEngine::Certificate } + fm::DiagnosisEngineKind::SmfService => DiagnosisEngine::SmfService, } } } diff --git a/nexus/db-model/src/fm/fact_smf_service.rs b/nexus/db-model/src/fm/fact_smf_service.rs new file mode 100644 index 00000000000..6b400dab9ba --- /dev/null +++ b/nexus/db-model/src/fm/fact_smf_service.rs @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database representation of the SMF service diagnosis engine's facts. +//! +//! Each SMF service fact is stored as typed columns in the +//! `fm_fact_smf_service` table. See [`nexus_types::fm::SmfServiceFact`] for +//! semantics. + +use crate::DbTypedUuid; +use crate::impl_enum_type; +use chrono::{DateTime, Utc}; +use nexus_db_schema::schema::fm_fact_smf_service; +use nexus_types::fm; +use nexus_types::fm::case::FactMetadata; +use nexus_types::fm::{FactPayload, SmfServiceFact, SmfServiceFactPayload}; +use omicron_uuid_kinds::{ + CaseKind, CollectionKind, FactKind, SitrepKind, SledKind, +}; + +impl_enum_type!( + FmFactSmfServiceKindEnum: + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq)] + pub enum FmFactSmfServiceKind; + + InMaintenance => b"in_maintenance" +); + +/// Diesel row for the `fm_fact_smf_service` table. +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = fm_fact_smf_service)] +pub struct FmFactSmfService { + pub id: DbTypedUuid, + /// The sitrep to which this fact belongs. + /// + /// This will change as the fact is carried forward from one sitrep to the + /// next. + pub sitrep_id: DbTypedUuid, + pub case_id: DbTypedUuid, + /// Sitrep in which this fact was first added. + /// + /// Preserved unchanged when the fact is carried forward; debug-only. + pub created_sitrep_id: DbTypedUuid, + pub comment: String, + + pub sled_id: DbTypedUuid, + pub zone_name: String, + pub fmri: String, + pub kind: FmFactSmfServiceKind, + + pub observed_in_inv: DbTypedUuid, + pub time_observed: DateTime, +} + +impl FmFactSmfService { + /// Build a row from a fact's shared metadata (`metadata`) and its + /// already-dispatched SMF service payload (`smf_fact`). + pub fn from_sitrep( + sitrep_id: impl Into>, + case_id: impl Into>, + metadata: &FactMetadata, + smf_fact: &SmfServiceFact, + ) -> Self { + let FactMetadata { id, created_sitrep_id, comment } = metadata; + let kind = match smf_fact { + SmfServiceFact::InMaintenance(_) => { + FmFactSmfServiceKind::InMaintenance + } + }; + let SmfServiceFactPayload { + sled_id, + zone_name, + fmri, + observed_in_inv, + time_observed, + } = smf_fact.payload(); + Self { + id: (*id).into(), + sitrep_id: sitrep_id.into(), + case_id: case_id.into(), + created_sitrep_id: (*created_sitrep_id).into(), + comment: comment.clone(), + sled_id: (*sled_id).into(), + zone_name: zone_name.clone(), + fmri: fmri.clone(), + kind, + observed_in_inv: (*observed_in_inv).into(), + time_observed: *time_observed, + } + } + + /// Reconstruct an in-memory fact from a row. + pub fn into_fact(self) -> fm::case::Fact { + let payload = SmfServiceFactPayload { + sled_id: self.sled_id.into(), + zone_name: self.zone_name, + fmri: self.fmri, + observed_in_inv: self.observed_in_inv.into(), + time_observed: self.time_observed, + }; + let payload = match self.kind { + FmFactSmfServiceKind::InMaintenance => { + FactPayload::SmfService(SmfServiceFact::InMaintenance(payload)) + } + }; + fm::case::Fact { + metadata: fm::case::FactMetadata { + id: self.id.into(), + created_sitrep_id: self.created_sitrep_id.into(), + comment: self.comment, + }, + payload, + } + } +} diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index c6143a754e1..579dbbd1999 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(312, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(313, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(313, "fm-smf-service-de"), KnownVersion::new(312, "inv-discovered-rack-and-sled"), KnownVersion::new(311, "instance-state-sled-evacuating"), KnownVersion::new(310, "add-unadopted-sled-and-rack-tables"), diff --git a/nexus/db-queries/src/db/datastore/fm.rs b/nexus/db-queries/src/db/datastore/fm.rs index 26163bc54c3..fda3a14ca7a 100644 --- a/nexus/db-queries/src/db/datastore/fm.rs +++ b/nexus/db-queries/src/db/datastore/fm.rs @@ -43,6 +43,7 @@ use nexus_db_schema::schema::fm_ereport_in_case::dsl as case_ereport_dsl; use nexus_db_schema::schema::fm_fact_certificate::dsl as fact_cert_dsl; use nexus_db_schema::schema::fm_fact_physical_disk::dsl as fact_pd_dsl; use nexus_db_schema::schema::fm_fact_saga::dsl as fact_saga_dsl; +use nexus_db_schema::schema::fm_fact_smf_service::dsl as fact_smf_dsl; use nexus_db_schema::schema::fm_sitrep::dsl as sitrep_dsl; use nexus_db_schema::schema::fm_sitrep_analysis_report::dsl as analysis_report_dsl; use nexus_db_schema::schema::fm_sitrep_history::dsl as history_dsl; @@ -139,6 +140,7 @@ sitrep_child_tables! { FmFactPhysicalDisk => { table: "fm_fact_physical_disk" }, FmFactSaga => { table: "fm_fact_saga" }, FmFactCertificate => { table: "fm_fact_certificate" }, + FmFactSmfService => { table: "fm_fact_smf_service" }, AnalysisReport => { table: "fm_sitrep_analysis_report" }, } @@ -669,6 +671,31 @@ impl DataStore { } } + // --- SMF service diagnosis engine facts --- + let mut paginator: Paginator> = + Paginator::new(SQL_BATCH_SIZE, PaginationOrder::Descending); + while let Some(p) = paginator.next() { + let batch = paginated( + fact_smf_dsl::fm_fact_smf_service, + fact_smf_dsl::id, + &p.current_pagparams(), + ) + .filter(fact_smf_dsl::sitrep_id.eq(id.into_untyped_uuid())) + .select(model::fm::FmFactSmfService::as_select()) + .load_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context("failed to load SMF service case facts") + })?; + + paginator = p.found_batch(&batch, &|f| f.id); + for row in batch { + let case_id: CaseUuid = row.case_id.into(); + insert_fact_for_case(&mut by_case, case_id, row.into_fact())?; + } + } + Ok(by_case) } @@ -930,6 +957,7 @@ impl DataStore { let mut physical_disk_facts = Vec::new(); let mut saga_facts = Vec::new(); let mut certificate_facts = Vec::new(); + let mut smf_service_facts = Vec::new(); for case in sitrep.cases { let case_id = case.id; cases.push(model::fm::CaseMetadata::from_sitrep(sitrep_id, &case)); @@ -984,6 +1012,16 @@ impl DataStore { ), ); } + fm::FactPayload::SmfService(smf_fact) => { + smf_service_facts.push( + model::fm::FmFactSmfService::from_sitrep( + sitrep_id, + case_id, + &fact.metadata, + smf_fact, + ), + ); + } } } } @@ -1081,6 +1119,19 @@ impl DataStore { })?; } + if !smf_service_facts.is_empty() { + diesel::insert_into(fact_smf_dsl::fm_fact_smf_service) + .values(smf_service_facts) + .execute_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + .internal_context( + "failed to insert SMF service case facts", + ) + })?; + } + if !cases.is_empty() { diesel::insert_into(case_dsl::fm_case) .values(cases) @@ -3137,6 +3188,49 @@ mod tests { "best certificate has expired", ); + // An SMF service case, exercising the fm_fact_smf_service + // read/write/GC paths. + let case7 = { + let mut facts = iddqd::IdOrdMap::new(); + facts + .insert_unique(fm::case::Fact { + metadata: fm::case::FactMetadata { + id: FactUuid::new_v4(), + created_sitrep_id: sitrep_id, + comment: "a representative in-maintenance fact" + .to_string(), + }, + payload: fm::FactPayload::SmfService( + fm::SmfServiceFact::InMaintenance( + fm::SmfServiceFactPayload { + sled_id: omicron_uuid_kinds::SledUuid::new_v4(), + zone_name: "oxz_ntp".to_string(), + fmri: "svc:/oxide/ntp:default".to_string(), + observed_in_inv: + omicron_uuid_kinds::CollectionUuid::new_v4( + ), + time_observed: omicron_common::now_db_precision( + ), + }, + ), + ), + }) + .unwrap(); + fm::Case { + id: omicron_uuid_kinds::CaseUuid::new_v4(), + metadata: fm::case::Metadata { + created_sitrep_id: sitrep_id, + closed_sitrep_id: None, + de: fm::DiagnosisEngineKind::SmfService, + comment: "an SMF service in maintenance".to_string(), + }, + ereports: iddqd::IdOrdMap::new(), + alerts_requested: iddqd::IdOrdMap::new(), + support_bundles_requested: iddqd::IdOrdMap::new(), + facts, + } + }; + let mut cases = iddqd::IdOrdMap::new(); cases.insert_unique(case1.clone()).expect("failed to insert case 1"); cases.insert_unique(case2.clone()).expect("failed to insert case 2"); @@ -3144,6 +3238,7 @@ mod tests { cases.insert_unique(case4).expect("failed to insert case 4"); cases.insert_unique(case5).expect("failed to insert case 5"); cases.insert_unique(case6).expect("failed to insert case 6"); + cases.insert_unique(case7).expect("failed to insert case 7"); let mut ereports_by_id = iddqd::IdOrdMap::new(); for case in cases.iter() { ereports_by_id @@ -3454,7 +3549,7 @@ mod tests { .get_result_async::(&*conn) .await .expect("failed to count cases before deletion"); - assert_eq!(cases_before, 6, "six cases should exist before deletion"); + assert_eq!(cases_before, 7, "seven cases should exist before deletion"); let case_ereports_before: i64 = case_ereport_dsl::fm_ereport_in_case .filter( diff --git a/nexus/db-schema/src/enums.rs b/nexus/db-schema/src/enums.rs index e27770c0292..7e390eaa128 100644 --- a/nexus/db-schema/src/enums.rs +++ b/nexus/db-schema/src/enums.rs @@ -63,6 +63,7 @@ define_enums! { FmFactPhysicalDiskKindEnum => "fm_fact_physical_disk_kind", FmFactSagaKindEnum => "fm_fact_saga_kind", FmFactSagaOrphanReasonEnum => "fm_fact_saga_orphan_reason", + FmFactSmfServiceKindEnum => "fm_fact_smf_service_kind", FmdInventoryErrorKindEnum => "fmd_inventory_error_kind", HwM2SlotEnum => "hw_m2_slot", HwPowerStateEnum => "hw_power_state", diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index d17b49a5d90..2f3d2ef93c3 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -3492,6 +3492,22 @@ table! { } } +table! { + fm_fact_smf_service (sitrep_id, id) { + id -> Uuid, + sitrep_id -> Uuid, + case_id -> Uuid, + created_sitrep_id -> Uuid, + comment -> Text, + sled_id -> Uuid, + zone_name -> Text, + fmri -> Text, + kind -> crate::enums::FmFactSmfServiceKindEnum, + observed_in_inv -> Uuid, + time_observed -> Timestamptz, + } +} + table! { fm_ereport_in_case (sitrep_id, id) { id -> Uuid, @@ -3513,6 +3529,8 @@ allow_tables_to_appear_in_same_query!(fm_sitrep, fm_fact_saga); allow_tables_to_appear_in_same_query!(fm_case, fm_fact_saga); allow_tables_to_appear_in_same_query!(fm_sitrep, fm_fact_certificate); allow_tables_to_appear_in_same_query!(fm_case, fm_fact_certificate); +allow_tables_to_appear_in_same_query!(fm_sitrep, fm_fact_smf_service); +allow_tables_to_appear_in_same_query!(fm_case, fm_fact_smf_service); table! { fm_alert_request (sitrep_id, id) { diff --git a/nexus/fm/Cargo.toml b/nexus/fm/Cargo.toml index 967e3d8e2e6..f2d15d15f9e 100644 --- a/nexus/fm/Cargo.toml +++ b/nexus/fm/Cargo.toml @@ -34,6 +34,7 @@ serde_json.workspace = true strum.workspace = true slog.workspace = true slog-error-chain.workspace = true +sled-agent-types.workspace = true steno.workspace = true thiserror.workspace = true typed-rng.workspace = true @@ -48,6 +49,7 @@ ereport-types = { workspace = true, optional = true } omicron-workspace-hack.workspace = true [dev-dependencies] +illumos-utils.workspace = true omicron-test-utils.workspace = true nexus-fm-slippy.workspace = true nexus-inventory.workspace = true diff --git a/nexus/fm/src/diagnosis/mod.rs b/nexus/fm/src/diagnosis/mod.rs index fa7f39f518c..6d730439823 100644 --- a/nexus/fm/src/diagnosis/mod.rs +++ b/nexus/fm/src/diagnosis/mod.rs @@ -13,12 +13,14 @@ mod certificate; mod physical_disk; mod power_shelf; mod saga; +mod smf_service; pub fn analyze(builder: &mut SitrepBuilder<'_>) -> anyhow::Result<()> { physical_disk::analyze(builder)?; power_shelf::analyze(builder)?; saga::analyze(builder)?; certificate::analyze(builder)?; + smf_service::analyze(builder)?; Ok(()) } diff --git a/nexus/fm/src/diagnosis/smf_service.rs b/nexus/fm/src/diagnosis/smf_service.rs new file mode 100644 index 00000000000..20d691db9e7 --- /dev/null +++ b/nexus/fm/src/diagnosis/smf_service.rs @@ -0,0 +1,732 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! SMF service diagnosis engine. +//! +//! Opens one case per SMF service instance (sled, zone, FMRI) that inventory +//! reports in the `maintenance` state. Other non-online states (`offline`, +//! `degraded`, unrecognized) are deliberately ignored, as are services in +//! Propolis zones. + +use crate::SitrepBuilder; +use iddqd::{BiHashItem, BiHashMap, bi_upcast}; +use nexus_types::fm; +use nexus_types::fm::DiagnosisEngineKind; +use nexus_types::fm::{SmfServiceFact, SmfServiceFactPayload}; +use nexus_types::inventory::is_propolis_zone; +use omicron_uuid_kinds::{CaseUuid, SledUuid}; +use sled_agent_types::inventory::SvcEnabledNotOnlineState; +use sled_agent_types::inventory::SvcsEnabledNotOnlineResult; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::fmt; + +/// Identifies one SMF service instance. Each case concerns exactly one. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct ServiceKey { + sled_id: SledUuid, + zone_name: String, + fmri: String, +} + +impl ServiceKey { + fn from_payload(p: &SmfServiceFactPayload) -> Self { + Self { + sled_id: p.sled_id, + zone_name: p.zone_name.clone(), + fmri: p.fmri.clone(), + } + } +} + +impl fmt::Display for ServiceKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { sled_id, zone_name, fmri } = self; + write!(f, "{fmri} in zone {zone_name} on sled {sled_id}") + } +} + +/// A parent-forwarded SMF service case, parsed into the form this engine +/// acts on. +struct ParsedCase { + case_id: CaseUuid, + service: ServiceKey, +} + +/// Keying on the service makes "at most one case per service" an invariant +/// of the map: inserting a second case for a service already present fails. +impl BiHashItem for ParsedCase { + type K1<'a> = CaseUuid; + type K2<'a> = &'a ServiceKey; + fn key1(&self) -> Self::K1<'_> { + self.case_id + } + fn key2(&self) -> Self::K2<'_> { + &self.service + } + bi_upcast!(); +} + +/// Why a parent-forwarded SMF service case could not be interpreted. +#[derive(Debug, Eq, PartialEq, thiserror::Error)] +enum UninterpretableCase { + #[error(transparent)] + ForeignFact(#[from] fm::case::ForeignFact), + #[error("facts reference different services ({expected} and {found})")] + DisagreeingServices { expected: ServiceKey, found: ServiceKey }, + #[error("case has no facts, so the service it concerns is unknown")] + NoFacts, +} + +fn parse_case(case: &fm::Case) -> Result { + let mut service: Option = None; + for fact in case.facts.iter() { + let SmfServiceFact::InMaintenance(payload) = fact.as_smf_service()?; + let found = ServiceKey::from_payload(payload); + match &service { + None => service = Some(found), + Some(expected) if *expected != found => { + return Err(UninterpretableCase::DisagreeingServices { + expected: expected.clone(), + found, + }); + } + Some(_) => {} + } + } + let service = service.ok_or(UninterpretableCase::NoFacts)?; + Ok(ParsedCase { case_id: case.id, service }) +} + +/// What the current inventory says about one sled's SMF services. +enum SledObservation { + /// The sled reported its services. `in_maintenance` holds the non-Propolis + /// services in maintenance, keyed by `(zone_name, fmri)`. + Observed { + in_maintenance: BTreeSet<(String, String)>, + /// False if some `svcs` output lines could not be parsed. + complete: bool, + time_of_status: chrono::DateTime, + }, + /// The sled is in inventory, but its service states are unknown. + Unknown, +} + +pub(super) fn analyze(builder: &mut SitrepBuilder<'_>) -> anyhow::Result<()> { + let input = builder.input(); + let inv = input.inventory(); + let in_service_sleds = input.in_service_sleds(); + + let observations: BTreeMap = inv + .sled_agents + .iter() + .map(|sa| { + let obs = match &sa.smf_services_enabled_not_online { + SvcsEnabledNotOnlineResult::SvcsEnabledNotOnline(svcs) => { + SledObservation::Observed { + in_maintenance: svcs + .services + .iter() + .filter(|svc| { + svc.state + == SvcEnabledNotOnlineState::Maintenance + && !is_propolis_zone(&svc.zone) + }) + .map(|svc| (svc.zone.clone(), svc.fmri.clone())) + .collect(), + complete: svcs.errors.is_empty(), + time_of_status: svcs.time_of_status, + } + } + SvcsEnabledNotOnlineResult::SvcsCmdError(_) + | SvcsEnabledNotOnlineResult::DataUnavailable => { + SledObservation::Unknown + } + }; + (sa.sled_id, obs) + }) + .collect(); + + for (sled_id, obs) in &observations { + if let SledObservation::Observed { complete: false, .. } = obs { + builder + .log_warning("sled reported unparseable SMF service status") + .kv("sled_id", sled_id) + .finish(); + } + } + + // Index the SMF service cases from the parent sitrep, closing any we + // cannot interpret and any duplicates. + let mut cases = BiHashMap::::new(); + for case in input + .open_cases() + .iter() + .filter(|c| c.metadata.de == DiagnosisEngineKind::SmfService) + { + let parsed = match parse_case(case) { + Ok(parsed) => parsed, + Err(reason) => { + builder + .log_warning("closing uninterpretable SMF service case") + .kv("case_id", case.id) + .kv("reason", reason.to_string()) + .finish(); + builder + .cases + .case_mut(&case.id) + .expect("case_id came from builder's open cases") + .close(format!("cannot interpret case: {reason}")); + continue; + } + }; + if let Err(dup) = cases.insert_unique(parsed) { + let kept = dup.duplicates()[0]; + builder + .cases + .case_mut(&case.id) + .expect("case_id came from builder's open cases") + .close(format!( + "duplicate of case {} for {}", + kept.case_id, kept.service, + )); + } + } + + // Close cases whose service has left maintenance or whose sled has left + // service. A sled missing from inventory, or whose service states are + // unknown, is not evidence of recovery. + for parsed in cases.iter() { + let service = &parsed.service; + let reason = if !in_service_sleds.contains(&service.sled_id) { + format!("sled {} no longer in service", service.sled_id) + } else { + match observations.get(&service.sled_id) { + // Unparseable lines could hide this service, so absence from + // a partial listing is not evidence of recovery. + Some(SledObservation::Observed { + in_maintenance, + complete: true, + .. + }) if !in_maintenance.contains(&( + service.zone_name.clone(), + service.fmri.clone(), + )) => + { + format!("{service} no longer reported in maintenance") + } + _ => continue, + } + }; + builder + .cases + .case_mut(&parsed.case_id) + .expect("case_id came from builder's open cases") + .close(reason); + } + + // Open a case for each service in maintenance on an in-service sled that + // doesn't already have one. Parsed maintenance lines are trustworthy even + // when other lines failed to parse, so this ignores `complete`. + for (sled_id, obs) in &observations { + if !in_service_sleds.contains(sled_id) { + continue; + } + let SledObservation::Observed { + in_maintenance, time_of_status, .. + } = obs + else { + continue; + }; + for (zone_name, fmri) in in_maintenance { + let service = ServiceKey { + sled_id: *sled_id, + zone_name: zone_name.clone(), + fmri: fmri.clone(), + }; + if cases.contains_key2(&&service) { + continue; + } + let mut case = builder.cases.open_case( + DiagnosisEngineKind::SmfService, + format!("opened because {service} is in maintenance"), + ); + case.add_fact( + SmfServiceFact::InMaintenance(SmfServiceFactPayload { + sled_id: *sled_id, + zone_name: zone_name.clone(), + fmri: fmri.clone(), + observed_in_inv: inv.id, + time_observed: *time_of_status, + }), + format!("{service} in maintenance"), + ); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis_input::Input; + use crate::test_util::{ + FmTest, make_parent_sitrep, make_zpool_degraded_fact, run_analyze, + }; + use chrono::Utc; + use iddqd::IdOrdMap; + use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; + use nexus_types::fm::Sitrep; + use nexus_types::inventory; + use omicron_test_utils::dev; + use omicron_uuid_kinds::{ + CollectionUuid, FactUuid, PhysicalDiskUuid, PropolisUuid, SitrepUuid, + ZpoolUuid, + }; + use sled_agent_types::inventory::{ + SvcEnabledNotOnline, SvcsEnabledNotOnline, SvcsError, + }; + use std::sync::Arc; + + const ZONE: &str = "oxz_ntp_aaaa"; + const FMRI: &str = "svc:/oxide/ntp:default"; + + /// Returns the example system's collection, with every sled reporting + /// no services in a non-online state, and the IDs of its sleds. + fn setup( + test_name: &'static str, + ) -> (dev::LogContext, inventory::Collection, Vec) { + let (fm_test, logctx) = FmTest::new_with_logctx(test_name); + let (example, _bp) = fm_test.system_builder.build(); + let mut collection = example.collection; + let sled_ids: Vec = + collection.sled_agents.iter().map(|sa| sa.sled_id).collect(); + assert!(sled_ids.len() >= 2, "example system should have 2+ sleds"); + for sled_id in &sled_ids { + set_services(&mut collection, *sled_id, &[], &[]); + } + (logctx, collection, sled_ids) + } + + fn set_result( + collection: &mut inventory::Collection, + sled_id: SledUuid, + result: SvcsEnabledNotOnlineResult, + ) { + collection + .sled_agents + .get_mut(sled_id) + .expect("sled is in collection") + .smf_services_enabled_not_online = result; + } + + fn set_services( + collection: &mut inventory::Collection, + sled_id: SledUuid, + services: &[(&str, &str, SvcEnabledNotOnlineState)], + errors: &[&str], + ) { + set_result( + collection, + sled_id, + SvcsEnabledNotOnlineResult::SvcsEnabledNotOnline( + SvcsEnabledNotOnline { + services: services + .iter() + .map(|(zone, fmri, state)| SvcEnabledNotOnline { + fmri: fmri.to_string(), + zone: zone.to_string(), + state: *state, + }) + .collect(), + errors: errors.iter().map(|e| e.to_string()).collect(), + time_of_status: Utc::now(), + }, + ), + ); + } + + fn set_maintenance( + collection: &mut inventory::Collection, + sled_id: SledUuid, + ) { + set_services( + collection, + sled_id, + &[(ZONE, FMRI, SvcEnabledNotOnlineState::Maintenance)], + &[], + ); + } + + fn build_input( + collection: inventory::Collection, + parent: Option, + in_service_sleds: impl IntoIterator, + ) -> Input { + let parent = parent.map(|s| { + Arc::new(( + fm::SitrepVersion { + id: s.id(), + version: 0, + time_made_current: Utc::now(), + }, + s, + )) + }); + let (input, _report) = Input::builder(parent, Arc::new(collection)) + .expect("input builder should accept inventory") + .in_service_sleds(Arc::new(in_service_sleds.into_iter().collect())) + .with_empty_defaults() + .build() + .expect("all inputs provided"); + input + } + + fn make_fact( + parent_sitrep_id: SitrepUuid, + inv_id: CollectionUuid, + sled_id: SledUuid, + fmri: &str, + ) -> fm::case::Fact { + fm::case::Fact { + metadata: fm::case::FactMetadata { + id: FactUuid::new_v4(), + created_sitrep_id: parent_sitrep_id, + comment: String::new(), + }, + payload: SmfServiceFact::InMaintenance(SmfServiceFactPayload { + sled_id, + zone_name: ZONE.to_string(), + fmri: fmri.to_string(), + observed_in_inv: inv_id, + time_observed: Utc::now(), + }) + .into(), + } + } + + fn make_case( + parent_sitrep_id: SitrepUuid, + facts: impl IntoIterator, + ) -> fm::Case { + let mut fact_map = IdOrdMap::new(); + for fact in facts { + fact_map.insert_unique(fact).unwrap(); + } + fm::Case { + id: CaseUuid::new_v4(), + metadata: fm::case::Metadata { + created_sitrep_id: parent_sitrep_id, + closed_sitrep_id: None, + de: DiagnosisEngineKind::SmfService, + comment: "an SMF service case".to_string(), + }, + ereports: Default::default(), + alerts_requested: Default::default(), + support_bundles_requested: Default::default(), + facts: fact_map, + } + } + + /// Returns a parent sitrep holding one open case for `FMRI` on + /// `sled_id`, and that case's ID. + fn parent_with_case( + collection: &inventory::Collection, + sled_id: SledUuid, + ) -> (Sitrep, CaseUuid) { + let parent_id = SitrepUuid::new_v4(); + let case = make_case( + parent_id, + [make_fact(parent_id, collection.id, sled_id, FMRI)], + ); + let case_id = case.id; + (make_parent_sitrep(parent_id, collection.id, [case]), case_id) + } + + fn smf_cases(sitrep: &Sitrep) -> Vec<&fm::Case> { + sitrep + .cases + .iter() + .filter(|c| c.metadata.de == DiagnosisEngineKind::SmfService) + .collect() + } + + fn open_smf_cases(sitrep: &Sitrep) -> Vec<&fm::Case> { + smf_cases(sitrep).into_iter().filter(|c| c.is_open()).collect() + } + + fn is_open(sitrep: &Sitrep, case_id: CaseUuid) -> bool { + sitrep.cases.get(&case_id).expect("case is in sitrep").is_open() + } + + #[test] + fn opens_case_for_service_in_maintenance() { + let (logctx, mut collection, sleds) = setup("smf_opens_case"); + set_maintenance(&mut collection, sleds[0]); + let inv_id = collection.id; + let input = build_input(collection, None, sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + let cases = open_smf_cases(&sitrep); + assert_eq!(cases.len(), 1); + let facts: Vec<_> = cases[0].facts.iter().collect(); + assert_eq!(facts.len(), 1); + let SmfServiceFact::InMaintenance(payload) = + facts[0].as_smf_service().unwrap(); + assert_eq!(payload.sled_id, sleds[0]); + assert_eq!(payload.zone_name, ZONE); + assert_eq!(payload.fmri, FMRI); + assert_eq!(payload.observed_in_inv, inv_id); + logctx.cleanup_successful(); + } + + #[test] + fn ignores_other_states_and_propolis_zones() { + let (logctx, mut collection, sleds) = setup("smf_ignores_others"); + let propolis_zone = + format!("{PROPOLIS_ZONE_PREFIX}{}", PropolisUuid::new_v4()); + set_services( + &mut collection, + sleds[0], + &[ + (ZONE, "svc:/a:default", SvcEnabledNotOnlineState::Offline), + (ZONE, "svc:/b:default", SvcEnabledNotOnlineState::Degraded), + ( + ZONE, + "svc:/c:default", + SvcEnabledNotOnlineState::Unrecognized, + ), + ( + &propolis_zone, + "svc:/d:default", + SvcEnabledNotOnlineState::Maintenance, + ), + ], + &[], + ); + let input = build_input(collection, None, sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(smf_cases(&sitrep).is_empty()); + logctx.cleanup_successful(); + } + + #[test] + fn does_not_open_case_on_sled_not_in_service() { + let (logctx, mut collection, sleds) = setup("smf_not_in_service"); + set_maintenance(&mut collection, sleds[0]); + let input = + build_input(collection, None, sleds.iter().copied().skip(1)); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(smf_cases(&sitrep).is_empty()); + logctx.cleanup_successful(); + } + + #[test] + fn keeps_case_while_still_in_maintenance() { + let (logctx, mut collection, sleds) = setup("smf_keeps_case"); + set_maintenance(&mut collection, sleds[0]); + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + let parent_case = parent.cases.get(&case_id).unwrap().clone(); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + let cases = open_smf_cases(&sitrep); + assert_eq!(cases.len(), 1); + assert_eq!(cases[0].id, case_id); + assert_eq!(cases[0].facts, parent_case.facts); + logctx.cleanup_successful(); + } + + #[test] + fn closes_case_when_service_recovers() { + let (logctx, collection, sleds) = setup("smf_closes_recovered"); + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(!is_open(&sitrep, case_id)); + assert!(open_smf_cases(&sitrep).is_empty()); + logctx.cleanup_successful(); + } + + #[test] + fn closes_case_when_sled_leaves_service() { + let (logctx, mut collection, sleds) = setup("smf_closes_expunged"); + set_maintenance(&mut collection, sleds[0]); + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + let input = build_input( + collection, + Some(parent), + sleds.iter().copied().skip(1), + ); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(!is_open(&sitrep, case_id)); + assert!(open_smf_cases(&sitrep).is_empty()); + logctx.cleanup_successful(); + } + + #[test] + fn closes_case_when_absent_sled_leaves_service() { + let (logctx, mut collection, sleds) = + setup("smf_closes_absent_expunged"); + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + collection.sled_agents.remove(sleds[0]); + let input = build_input( + collection, + Some(parent), + sleds.iter().copied().skip(1), + ); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(!is_open(&sitrep, case_id)); + logctx.cleanup_successful(); + } + + #[test] + fn keeps_case_when_status_unknown() { + type MakeUnknown = fn(&mut inventory::Collection, SledUuid); + let unknown: [(&str, MakeUnknown); 3] = [ + ("smf_unknown_absent", |c, s| { + c.sled_agents.remove(s); + }), + ("smf_unknown_data_unavailable", |c, s| { + set_result(c, s, SvcsEnabledNotOnlineResult::DataUnavailable) + }), + ("smf_unknown_cmd_error", |c, s| { + set_result( + c, + s, + SvcsEnabledNotOnlineResult::SvcsCmdError(SvcsError { + error: "svcs failed".to_string(), + time_of_status: Utc::now(), + }), + ) + }), + ]; + for (test_name, make_unknown) in unknown { + let (logctx, mut collection, sleds) = setup(test_name); + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + make_unknown(&mut collection, sleds[0]); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(is_open(&sitrep, case_id), "{test_name}"); + assert_eq!(open_smf_cases(&sitrep).len(), 1, "{test_name}"); + logctx.cleanup_successful(); + } + } + + #[test] + fn partial_listing_opens_but_does_not_close() { + let (logctx, mut collection, sleds) = setup("smf_partial_listing"); + // The parent has a case for FMRI, which is absent from this partial + // listing; a different service is newly in maintenance. + let (parent, case_id) = parent_with_case(&collection, sleds[0]); + let new_fmri = "svc:/oxide/other:default"; + set_services( + &mut collection, + sleds[0], + &[(ZONE, new_fmri, SvcEnabledNotOnlineState::Maintenance)], + &["Unexpected output line: maintenance"], + ); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, report) = run_analyze(&logctx.log, &input); + assert!(is_open(&sitrep, case_id)); + let fmris: BTreeSet<_> = open_smf_cases(&sitrep) + .iter() + .flat_map(|c| c.facts.iter()) + .map(|f| f.as_smf_service().unwrap().payload().fmri.clone()) + .collect(); + assert_eq!( + fmris, + BTreeSet::from([FMRI.to_string(), new_fmri.to_string()]) + ); + let report = format!("{report:?}"); + assert!( + report.contains("unparseable SMF service status"), + "expected a warning in the report: {report}" + ); + logctx.cleanup_successful(); + } + + #[test] + fn partial_listing_only_blocks_closing_on_its_own_sled() { + let (logctx, mut collection, sleds) = setup("smf_partial_per_sled"); + let parent_id = SitrepUuid::new_v4(); + let inv_id = collection.id; + let case_a = make_case( + parent_id, + [make_fact(parent_id, inv_id, sleds[0], FMRI)], + ); + let case_b = make_case( + parent_id, + [make_fact(parent_id, inv_id, sleds[1], FMRI)], + ); + let (id_a, id_b) = (case_a.id, case_b.id); + let parent = make_parent_sitrep(parent_id, inv_id, [case_a, case_b]); + set_services(&mut collection, sleds[0], &[], &["bad line"]); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + assert!(is_open(&sitrep, id_a)); + assert!(!is_open(&sitrep, id_b)); + logctx.cleanup_successful(); + } + + #[test] + fn closes_duplicate_and_uninterpretable_cases() { + let (logctx, mut collection, sleds) = setup("smf_bad_cases"); + set_maintenance(&mut collection, sleds[0]); + let parent_id = SitrepUuid::new_v4(); + let inv_id = collection.id; + let first = make_case( + parent_id, + [make_fact(parent_id, inv_id, sleds[0], FMRI)], + ); + let duplicate = make_case( + parent_id, + [make_fact(parent_id, inv_id, sleds[0], FMRI)], + ); + let empty = make_case(parent_id, []); + let foreign = make_case( + parent_id, + [make_zpool_degraded_fact( + parent_id, + inv_id, + PhysicalDiskUuid::new_v4(), + ZpoolUuid::new_v4(), + )], + ); + let disagreeing = make_case( + parent_id, + [ + make_fact(parent_id, inv_id, sleds[0], "svc:/x:default"), + make_fact(parent_id, inv_id, sleds[1], "svc:/x:default"), + ], + ); + let ids = + [first.id, duplicate.id, empty.id, disagreeing.id, foreign.id]; + let parent = make_parent_sitrep( + parent_id, + inv_id, + [first, duplicate, empty, disagreeing, foreign], + ); + let input = build_input(collection, Some(parent), sleds.clone()); + + let (sitrep, _) = run_analyze(&logctx.log, &input); + let open: Vec<_> = + open_smf_cases(&sitrep).iter().map(|c| c.id).collect(); + // Open cases are visited in case ID order, so the lower ID survives. + assert_eq!(open, vec![ids[0].min(ids[1])]); + assert!(!is_open(&sitrep, ids[2])); + assert!(!is_open(&sitrep, ids[3])); + assert!(!is_open(&sitrep, ids[4])); + logctx.cleanup_successful(); + } +} diff --git a/nexus/fm/src/test_util.rs b/nexus/fm/src/test_util.rs index 7380039351c..7973673d93b 100644 --- a/nexus/fm/src/test_util.rs +++ b/nexus/fm/src/test_util.rs @@ -82,6 +82,7 @@ impl FmTest { // TODO(eliza): eventually it would be nice if the inventory collection and // in-service-disks were generated from the `ExampleSystemBuilder` // somehow... + #[allow(clippy::too_many_arguments)] pub fn input_builder( &self, parent_sitrep: Option>, diff --git a/nexus/types/src/fm.rs b/nexus/types/src/fm.rs index 51ab04569b1..fb3655958e7 100644 --- a/nexus/types/src/fm.rs +++ b/nexus/types/src/fm.rs @@ -21,7 +21,8 @@ pub mod fact; pub use fact::{ CertificateExpiryFactPayload, CertificateFact, DiskFact, FactPayload, SagaAbandonedFactPayload, SagaFact, SagaNotProgressingFactPayload, - SagaOwnerNotCurrentFactPayload, ZpoolUnhealthyFactPayload, + SagaOwnerNotCurrentFactPayload, SmfServiceFact, SmfServiceFactPayload, + ZpoolUnhealthyFactPayload, }; pub mod display; @@ -279,4 +280,5 @@ pub enum DiagnosisEngineKind { PhysicalDisk, Saga, Certificate, + SmfService, } diff --git a/nexus/types/src/fm/case.rs b/nexus/types/src/fm/case.rs index f0fa8a7e460..bea04bcc75f 100644 --- a/nexus/types/src/fm/case.rs +++ b/nexus/types/src/fm/case.rs @@ -9,7 +9,7 @@ use crate::fm::DiagnosisEngineKind; use crate::fm::Ereport; use crate::fm::EreportId; use crate::fm::FactPayload; -use crate::fm::{CertificateFact, DiskFact, SagaFact}; +use crate::fm::{CertificateFact, DiskFact, SagaFact, SmfServiceFact}; use crate::support_bundle::BundleDataSelection; use iddqd::{IdOrdItem, IdOrdMap}; use omicron_uuid_kinds::{ @@ -231,6 +231,12 @@ impl Fact { self.payload.as_certificate().ok_or_else(|| self.foreign_fact()) } + /// The SMF service payload, or a [`ForeignFact`] error if this fact + /// belongs to a different diagnosis engine. + pub fn as_smf_service(&self) -> Result<&SmfServiceFact, ForeignFact> { + self.payload.as_smf_service().ok_or_else(|| self.foreign_fact()) + } + fn foreign_fact(&self) -> ForeignFact { ForeignFact { fact_id: self.metadata.id, actual: self.payload.engine() } } diff --git a/nexus/types/src/fm/fact.rs b/nexus/types/src/fm/fact.rs index bfd6bcd922b..d8d0f5edc86 100644 --- a/nexus/types/src/fm/fact.rs +++ b/nexus/types/src/fm/fact.rs @@ -18,7 +18,7 @@ use crate::observed_saga::{OrphanedReason, SagaProgressState}; use chrono::{DateTime, Utc}; use omicron_uuid_kinds::SiloUuid; use omicron_uuid_kinds::{ - CollectionUuid, OmicronZoneUuid, PhysicalDiskUuid, ZpoolUuid, + CollectionUuid, OmicronZoneUuid, PhysicalDiskUuid, SledUuid, ZpoolUuid, }; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -37,6 +37,8 @@ pub enum FactPayload { Saga(SagaFact), /// A fact owned by the certificate diagnosis engine. Certificate(CertificateFact), + /// A fact owned by the SMF service diagnosis engine. + SmfService(SmfServiceFact), } impl From for FactPayload { @@ -57,6 +59,12 @@ impl From for FactPayload { } } +impl From for FactPayload { + fn from(fact: SmfServiceFact) -> Self { + FactPayload::SmfService(fact) + } +} + impl FactPayload { /// The diagnosis engine that owns this payload's variant. pub fn engine(&self) -> DiagnosisEngineKind { @@ -64,6 +72,7 @@ impl FactPayload { FactPayload::PhysicalDisk(_) => DiagnosisEngineKind::PhysicalDisk, FactPayload::Saga(_) => DiagnosisEngineKind::Saga, FactPayload::Certificate(_) => DiagnosisEngineKind::Certificate, + FactPayload::SmfService(_) => DiagnosisEngineKind::SmfService, } } @@ -93,6 +102,15 @@ impl FactPayload { _ => None, } } + + /// The SMF service payload, or `None` if this fact belongs to a + /// different diagnosis engine. + pub fn as_smf_service(&self) -> Option<&SmfServiceFact> { + match self { + FactPayload::SmfService(fact) => Some(fact), + _ => None, + } + } } /// Per-fact state for the physical-disk diagnosis engine. @@ -274,3 +292,40 @@ pub struct CertificateExpiryFactPayload { /// The leaf `not_after` of that certificate. pub not_after: DateTime, } + +/// Per-fact state for the SMF service diagnosis engine. +/// +/// An SMF service case is keyed by the service instance: its sled, zone, and +/// FMRI. Only the `maintenance` state is tracked. Services that are offline, +/// degraded, or in an unrecognized state are deliberately ignored. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SmfServiceFact { + /// The service instance is enabled, but in the `maintenance` state. + InMaintenance(SmfServiceFactPayload), +} + +impl SmfServiceFact { + /// The condition-defining payload, shared by every kind. + pub fn payload(&self) -> &SmfServiceFactPayload { + match self { + SmfServiceFact::InMaintenance(p) => p, + } + } +} + +/// Payload of an [`SmfServiceFact`]. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SmfServiceFactPayload { + /// The sled the service runs on. + pub sled_id: SledUuid, + /// The zone the service runs in (`global` for the global zone). + pub zone_name: String, + /// The service instance's FMRI. + pub fmri: String, + /// Inventory collection that produced this observation. Recorded for + /// provenance only; see [`ZpoolUnhealthyFactPayload::observed_in_inv`]. + pub observed_in_inv: CollectionUuid, + /// When the sled agent observed the service in this state. + pub time_observed: DateTime, +} diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1ac174add91..2c86ab8d0ef 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -8513,7 +8513,8 @@ CREATE TYPE IF NOT EXISTS omicron.public.diagnosis_engine AS ENUM ( 'power_shelf', 'physical_disk', 'saga', - 'certificate' + 'certificate', + 'smf_service' ); CREATE TABLE IF NOT EXISTS omicron.public.fm_case ( @@ -8702,6 +8703,45 @@ CREATE TABLE IF NOT EXISTS omicron.public.fm_fact_certificate ( PRIMARY KEY (sitrep_id, id) ); +-- The SMF service diagnosis engine's facts. See the comment on the +-- physical-disk engine above: one table per engine, fact content as typed +-- columns. +CREATE TYPE IF NOT EXISTS omicron.public.fm_fact_smf_service_kind AS ENUM ( + 'in_maintenance' +); + +CREATE TABLE IF NOT EXISTS omicron.public.fm_fact_smf_service ( + -- Stable UUID for this fact across sitreps. + id UUID NOT NULL, + -- Sitrep this row belongs to. + sitrep_id UUID NOT NULL, + -- UUID of the case this fact attaches to. + case_id UUID NOT NULL, + -- UUID of the sitrep in which this fact was first added. Preserved + -- unchanged when the fact is carried forward into a child sitrep. + -- Debug-only. + created_sitrep_id UUID NOT NULL, + -- Free-form, debug-only comment. + comment TEXT NOT NULL, + + -- The SMF service instance this fact is about. Common to every kind of + -- SMF service fact (the case is keyed by it), so it is always present + -- regardless of `kind`. + sled_id UUID NOT NULL, + zone_name TEXT NOT NULL, + fmri TEXT NOT NULL, + + -- Which SMF service fact this row represents. + kind omicron.public.fm_fact_smf_service_kind NOT NULL, + + -- The inventory collection in which the service was observed, and when + -- the sled agent observed it. + observed_in_inv UUID NOT NULL, + time_observed TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (sitrep_id, id) +); + CREATE TABLE IF NOT EXISTS omicron.public.fm_ereport_in_case ( -- ID of this association. When an ereport is assigned to a case, that -- association is assigned a UUID. These are used primarily to aid in @@ -9882,7 +9922,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '312.0.0', NULL) + (TRUE, NOW(), NOW(), '313.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/fm-smf-service-de/up1.sql b/schema/crdb/fm-smf-service-de/up1.sql new file mode 100644 index 00000000000..9b50b60465c --- /dev/null +++ b/schema/crdb/fm-smf-service-de/up1.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.diagnosis_engine ADD VALUE IF NOT EXISTS 'smf_service' AFTER 'certificate'; diff --git a/schema/crdb/fm-smf-service-de/up2.sql b/schema/crdb/fm-smf-service-de/up2.sql new file mode 100644 index 00000000000..fde39e73d98 --- /dev/null +++ b/schema/crdb/fm-smf-service-de/up2.sql @@ -0,0 +1,3 @@ +CREATE TYPE IF NOT EXISTS omicron.public.fm_fact_smf_service_kind AS ENUM ( + 'in_maintenance' +); diff --git a/schema/crdb/fm-smf-service-de/up3.sql b/schema/crdb/fm-smf-service-de/up3.sql new file mode 100644 index 00000000000..195dff6bb00 --- /dev/null +++ b/schema/crdb/fm-smf-service-de/up3.sql @@ -0,0 +1,31 @@ +CREATE TABLE IF NOT EXISTS omicron.public.fm_fact_smf_service ( + -- Stable UUID for this fact across sitreps. + id UUID NOT NULL, + -- Sitrep this row belongs to. + sitrep_id UUID NOT NULL, + -- UUID of the case this fact attaches to. + case_id UUID NOT NULL, + -- UUID of the sitrep in which this fact was first added. Preserved + -- unchanged when the fact is carried forward into a child sitrep. + -- Debug-only. + created_sitrep_id UUID NOT NULL, + -- Free-form, debug-only comment. + comment TEXT NOT NULL, + + -- The SMF service instance this fact is about. Common to every kind of + -- SMF service fact (the case is keyed by it), so it is always present + -- regardless of `kind`. + sled_id UUID NOT NULL, + zone_name TEXT NOT NULL, + fmri TEXT NOT NULL, + + -- Which SMF service fact this row represents. + kind omicron.public.fm_fact_smf_service_kind NOT NULL, + + -- The inventory collection in which the service was observed, and when + -- the sled agent observed it. + observed_in_inv UUID NOT NULL, + time_observed TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (sitrep_id, id) +); From 1deb962ca98abd1634a3386b2f8a7cb3395f8f31 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 5 Nov 2026 14:04:01 -0800 Subject: [PATCH 4/4] simplify fm_fact_smf_service comment in dbinit.sql --- schema/crdb/dbinit.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 2c86ab8d0ef..70ac8d1f22f 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -8703,9 +8703,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.fm_fact_certificate ( PRIMARY KEY (sitrep_id, id) ); --- The SMF service diagnosis engine's facts. See the comment on the --- physical-disk engine above: one table per engine, fact content as typed --- columns. +-- The SMF service diagnosis engine's facts. CREATE TYPE IF NOT EXISTS omicron.public.fm_fact_smf_service_kind AS ENUM ( 'in_maintenance' );