[fm] add SMF service diagnosis engine for services in maintenance

dev-tools/omdb/tests/successes.out

@@ -841,6 +841,7 @@ task: "fm_analysis"
       ereporter_restarts:     <REDACTED DURATION>ms
       ereports:               <REDACTED DURATION>ms
       in_service_disks:       <REDACTED DURATION>ms
+      in_service_sleds:       <REDACTED DURATION>ms
       observed_sagas:         <REDACTED DURATION>ms
       silo_certificates:      <REDACTED DURATION>ms
       support_bundle_markers: <REDACTED DURATION>ms
@@ -917,6 +918,8 @@ task: "fm_sitrep_gc"
       batches:                                                                    1
     orphaned fm_fact_saga rows deleted:                                           0
       batches:                                                                    1
+    orphaned fm_fact_smf_service rows deleted:                                    0
+      batches:                                                                    1
     orphaned fm_sitrep_analysis_report rows deleted:                              <ORPHANED_FMSITREPANALYSISREPORT_ROWS_DELETED_REDACTED>
       batches:                                                                    1
     orphaned fm_support_bundle_request rows deleted:                              0
@@ -1604,6 +1607,7 @@ task: "fm_analysis"
       ereporter_restarts:     <REDACTED DURATION>ms
       ereports:               <REDACTED DURATION>ms
       in_service_disks:       <REDACTED DURATION>ms
+      in_service_sleds:       <REDACTED DURATION>ms
       observed_sagas:         <REDACTED DURATION>ms
       silo_certificates:      <REDACTED DURATION>ms
       support_bundle_markers: <REDACTED DURATION>ms
@@ -1680,6 +1684,8 @@ task: "fm_sitrep_gc"
       batches:                                                                    1
     orphaned fm_fact_saga rows deleted:                                           0
       batches:                                                                    1
+    orphaned fm_fact_smf_service rows deleted:                                    0
+      batches:                                                                    1
     orphaned fm_sitrep_analysis_report rows deleted:                              <ORPHANED_FMSITREPANALYSISREPORT_ROWS_DELETED_REDACTED>
       batches:                                                                    1
     orphaned fm_support_bundle_request rows deleted:                              0

nexus/db-model/src/fm.rs

@@ -38,6 +38,8 @@ mod fact_physical_disk;
 pub use fact_physical_disk::*;
 mod fact_saga;
 pub use fact_saga::*;
+mod fact_smf_service;
+pub use fact_smf_service::*;
 mod rendezvous_created;
 pub use rendezvous_created::*;
 mod support_bundle_request;

nexus/db-model/src/fm/diagnosis_engine.rs

@@ -27,6 +27,7 @@ impl_enum_type!(
     PhysicalDisk => b"physical_disk"
     Saga => b"saga"
     Certificate => b"certificate"
+    SmfService => b"smf_service"
 
 );
 
@@ -41,6 +42,7 @@ impl From<DiagnosisEngine> for fm::DiagnosisEngineKind {
             DiagnosisEngine::Certificate => {
                 fm::DiagnosisEngineKind::Certificate
             }
+            DiagnosisEngine::SmfService => fm::DiagnosisEngineKind::SmfService,
         }
     }
 }
@@ -56,6 +58,7 @@ impl From<fm::DiagnosisEngineKind> for DiagnosisEngine {
             fm::DiagnosisEngineKind::Certificate => {
                 DiagnosisEngine::Certificate
             }
+            fm::DiagnosisEngineKind::SmfService => DiagnosisEngine::SmfService,
         }
     }
 }

nexus/db-model/src/fm/fact_smf_service.rs

@@ -0,0 +1,117 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database representation of the SMF service diagnosis engine's facts.
+//!
+//! Each SMF service fact is stored as typed columns in the
+//! `fm_fact_smf_service` table. See [`nexus_types::fm::SmfServiceFact`] for
+//! semantics.
+
+use crate::DbTypedUuid;
+use crate::impl_enum_type;
+use chrono::{DateTime, Utc};
+use nexus_db_schema::schema::fm_fact_smf_service;
+use nexus_types::fm;
+use nexus_types::fm::case::FactMetadata;
+use nexus_types::fm::{FactPayload, SmfServiceFact, SmfServiceFactPayload};
+use omicron_uuid_kinds::{
+    CaseKind, CollectionKind, FactKind, SitrepKind, SledKind,
+};
+
+impl_enum_type!(
+    FmFactSmfServiceKindEnum:
+
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq)]
+    pub enum FmFactSmfServiceKind;
+
+    InMaintenance => b"in_maintenance"
+);
+
+/// Diesel row for the `fm_fact_smf_service` table.
+#[derive(Queryable, Insertable, Clone, Debug, Selectable)]
+#[diesel(table_name = fm_fact_smf_service)]
+pub struct FmFactSmfService {
+    pub id: DbTypedUuid<FactKind>,
+    /// The sitrep to which this fact belongs.
+    ///
+    /// This will change as the fact is carried forward from one sitrep to the
+    /// next.
+    pub sitrep_id: DbTypedUuid<SitrepKind>,
+    pub case_id: DbTypedUuid<CaseKind>,
+    /// Sitrep in which this fact was first added.
+    ///
+    /// Preserved unchanged when the fact is carried forward; debug-only.
+    pub created_sitrep_id: DbTypedUuid<SitrepKind>,
+    pub comment: String,
+
+    pub sled_id: DbTypedUuid<SledKind>,
+    pub zone_name: String,
+    pub fmri: String,
+    pub kind: FmFactSmfServiceKind,
+
+    pub observed_in_inv: DbTypedUuid<CollectionKind>,
+    pub time_observed: DateTime<Utc>,
+}
+
+impl FmFactSmfService {
+    /// Build a row from a fact's shared metadata (`metadata`) and its
+    /// already-dispatched SMF service payload (`smf_fact`).
+    pub fn from_sitrep(
+        sitrep_id: impl Into<DbTypedUuid<SitrepKind>>,
+        case_id: impl Into<DbTypedUuid<CaseKind>>,
+        metadata: &FactMetadata,
+        smf_fact: &SmfServiceFact,
+    ) -> Self {
+        let FactMetadata { id, created_sitrep_id, comment } = metadata;
+        let kind = match smf_fact {
+            SmfServiceFact::InMaintenance(_) => {
+                FmFactSmfServiceKind::InMaintenance
+            }
+        };
+        let SmfServiceFactPayload {
+            sled_id,
+            zone_name,
+            fmri,
+            observed_in_inv,
+            time_observed,
+        } = smf_fact.payload();
+        Self {
+            id: (*id).into(),
+            sitrep_id: sitrep_id.into(),
+            case_id: case_id.into(),
+            created_sitrep_id: (*created_sitrep_id).into(),
+            comment: comment.clone(),
+            sled_id: (*sled_id).into(),
+            zone_name: zone_name.clone(),
+            fmri: fmri.clone(),
+            kind,
+            observed_in_inv: (*observed_in_inv).into(),
+            time_observed: *time_observed,
+        }
+    }
+
+    /// Reconstruct an in-memory fact from a row.
+    pub fn into_fact(self) -> fm::case::Fact {
+        let payload = SmfServiceFactPayload {
+            sled_id: self.sled_id.into(),
+            zone_name: self.zone_name,
+            fmri: self.fmri,
+            observed_in_inv: self.observed_in_inv.into(),
+            time_observed: self.time_observed,
+        };
+        let payload = match self.kind {
+            FmFactSmfServiceKind::InMaintenance => {
+                FactPayload::SmfService(SmfServiceFact::InMaintenance(payload))
+            }
+        };
+        fm::case::Fact {
+            metadata: fm::case::FactMetadata {
+                id: self.id.into(),
+                created_sitrep_id: self.created_sitrep_id.into(),
+                comment: self.comment,
+            },
+            payload,
+        }
+    }
+}

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(312, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(313, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(313, "fm-smf-service-de"),
         KnownVersion::new(312, "inv-discovered-rack-and-sled"),
         KnownVersion::new(311, "instance-state-sled-evacuating"),
         KnownVersion::new(310, "add-unadopted-sled-and-rack-tables"),

nexus/db-queries/src/db/datastore/fm.rs

@@ -43,6 +43,7 @@ use nexus_db_schema::schema::fm_ereport_in_case::dsl as case_ereport_dsl;
 use nexus_db_schema::schema::fm_fact_certificate::dsl as fact_cert_dsl;
 use nexus_db_schema::schema::fm_fact_physical_disk::dsl as fact_pd_dsl;
 use nexus_db_schema::schema::fm_fact_saga::dsl as fact_saga_dsl;
+use nexus_db_schema::schema::fm_fact_smf_service::dsl as fact_smf_dsl;
 use nexus_db_schema::schema::fm_sitrep::dsl as sitrep_dsl;
 use nexus_db_schema::schema::fm_sitrep_analysis_report::dsl as analysis_report_dsl;
 use nexus_db_schema::schema::fm_sitrep_history::dsl as history_dsl;
@@ -139,6 +140,7 @@ sitrep_child_tables! {
     FmFactPhysicalDisk => { table: "fm_fact_physical_disk" },
     FmFactSaga => { table: "fm_fact_saga" },
     FmFactCertificate => { table: "fm_fact_certificate" },
+    FmFactSmfService => { table: "fm_fact_smf_service" },
     AnalysisReport => { table: "fm_sitrep_analysis_report" },
 }
 
@@ -669,6 +671,31 @@ impl DataStore {
             }
         }
 
+        // --- SMF service diagnosis engine facts ---
+        let mut paginator: Paginator<DbTypedUuid<FactKind>> =
+            Paginator::new(SQL_BATCH_SIZE, PaginationOrder::Descending);
+        while let Some(p) = paginator.next() {
+            let batch = paginated(
+                fact_smf_dsl::fm_fact_smf_service,
+                fact_smf_dsl::id,
+                &p.current_pagparams(),
+            )
+            .filter(fact_smf_dsl::sitrep_id.eq(id.into_untyped_uuid()))
+            .select(model::fm::FmFactSmfService::as_select())
+            .load_async(conn)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(e, ErrorHandler::Server)
+                    .internal_context("failed to load SMF service case facts")
+            })?;
+
+            paginator = p.found_batch(&batch, &|f| f.id);
+            for row in batch {
+                let case_id: CaseUuid = row.case_id.into();
+                insert_fact_for_case(&mut by_case, case_id, row.into_fact())?;
+            }
+        }
+
         Ok(by_case)
     }
 
@@ -930,6 +957,7 @@ impl DataStore {
         let mut physical_disk_facts = Vec::new();
         let mut saga_facts = Vec::new();
         let mut certificate_facts = Vec::new();
+        let mut smf_service_facts = Vec::new();
         for case in sitrep.cases {
             let case_id = case.id;
             cases.push(model::fm::CaseMetadata::from_sitrep(sitrep_id, &case));
@@ -984,6 +1012,16 @@ impl DataStore {
                             ),
                         );
                     }
+                    fm::FactPayload::SmfService(smf_fact) => {
+                        smf_service_facts.push(
+                            model::fm::FmFactSmfService::from_sitrep(
+                                sitrep_id,
+                                case_id,
+                                &fact.metadata,
+                                smf_fact,
+                            ),
+                        );
+                    }
                 }
             }
         }
@@ -1081,6 +1119,19 @@ impl DataStore {
                 })?;
         }
 
+        if !smf_service_facts.is_empty() {
+            diesel::insert_into(fact_smf_dsl::fm_fact_smf_service)
+                .values(smf_service_facts)
+                .execute_async(&*conn)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                        .internal_context(
+                            "failed to insert SMF service case facts",
+                        )
+                })?;
+        }
+
         if !cases.is_empty() {
             diesel::insert_into(case_dsl::fm_case)
                 .values(cases)
@@ -3137,13 +3188,57 @@ mod tests {
             "best certificate has expired",
         );
 
+        // An SMF service case, exercising the fm_fact_smf_service
+        // read/write/GC paths.
+        let case7 = {
+            let mut facts = iddqd::IdOrdMap::new();
+            facts
+                .insert_unique(fm::case::Fact {
+                    metadata: fm::case::FactMetadata {
+                        id: FactUuid::new_v4(),
+                        created_sitrep_id: sitrep_id,
+                        comment: "a representative in-maintenance fact"
+                            .to_string(),
+                    },
+                    payload: fm::FactPayload::SmfService(
+                        fm::SmfServiceFact::InMaintenance(
+                            fm::SmfServiceFactPayload {
+                                sled_id: omicron_uuid_kinds::SledUuid::new_v4(),
+                                zone_name: "oxz_ntp".to_string(),
+                                fmri: "svc:/oxide/ntp:default".to_string(),
+                                observed_in_inv:
+                                    omicron_uuid_kinds::CollectionUuid::new_v4(
+                                    ),
+                                time_observed: omicron_common::now_db_precision(
+                                ),
+                            },
+                        ),
+                    ),
+                })
+                .unwrap();
+            fm::Case {
+                id: omicron_uuid_kinds::CaseUuid::new_v4(),
+                metadata: fm::case::Metadata {
+                    created_sitrep_id: sitrep_id,
+                    closed_sitrep_id: None,
+                    de: fm::DiagnosisEngineKind::SmfService,
+                    comment: "an SMF service in maintenance".to_string(),
+                },
+                ereports: iddqd::IdOrdMap::new(),
+                alerts_requested: iddqd::IdOrdMap::new(),
+                support_bundles_requested: iddqd::IdOrdMap::new(),
+                facts,
+            }
+        };
+
         let mut cases = iddqd::IdOrdMap::new();
         cases.insert_unique(case1.clone()).expect("failed to insert case 1");
         cases.insert_unique(case2.clone()).expect("failed to insert case 2");
         cases.insert_unique(case3).expect("failed to insert case 3");
         cases.insert_unique(case4).expect("failed to insert case 4");
         cases.insert_unique(case5).expect("failed to insert case 5");
         cases.insert_unique(case6).expect("failed to insert case 6");
+        cases.insert_unique(case7).expect("failed to insert case 7");
         let mut ereports_by_id = iddqd::IdOrdMap::new();
         for case in cases.iter() {
             ereports_by_id
@@ -3193,6 +3288,7 @@ mod tests {
             open_cases: Default::default(),
             closed_cases_copied_forward: Default::default(),
             in_service_disks: Default::default(),
+            in_service_sleds: Default::default(),
             observed_sagas: Default::default(),
             observed_silo_certificates: Default::default(),
             config: None,
@@ -3453,7 +3549,7 @@ mod tests {
             .get_result_async::<i64>(&*conn)
             .await
             .expect("failed to count cases before deletion");
-        assert_eq!(cases_before, 6, "six cases should exist before deletion");
+        assert_eq!(cases_before, 7, "seven cases should exist before deletion");
 
         let case_ereports_before: i64 = case_ereport_dsl::fm_ereport_in_case
             .filter(
@@ -3836,6 +3932,7 @@ mod tests {
                 num_ereporter_restarts: 0,
                 unknown_restart_ereport_ids: Default::default(),
                 in_service_disks: Default::default(),
+                in_service_sleds: Default::default(),
                 observed_sagas: Default::default(),
                 observed_silo_certificates: Default::default(),
                 config: None,