[reconfigurator] background task for stopping marked VMMs

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -1296,6 +1296,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "abandoned_vmm_reaper" => {
             print_task_abandoned_vmm_reaper(details);
         }
+        "vmm_stop_for_update" => {
+            print_task_vmm_stop_for_update(details);
+        }
         "attached_subnet_manager" => {
             print_task_attached_subnet_manager_status(details);
         }
@@ -1510,6 +1513,58 @@ fn print_task_abandoned_vmm_reaper(details: &serde_json::Value) {
     };
 }
 
+fn print_task_vmm_stop_for_update(details: &serde_json::Value) {
+    use nexus_types::internal_api::background::VmmStopForUpdateStatus;
+
+    match serde_json::from_value::<VmmStopForUpdateStatus>(details.clone()) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+        Ok(VmmStopForUpdateStatus {
+            vmms_stopped_by_sled,
+            vmms_failed_to_stop_by_sled,
+            error_messages,
+        }) => {
+            let total: usize =
+                vmms_stopped_by_sled.iter().map(|s| s.vmm_ids.len()).sum();
+            println!("    VMMs stopped for sled evacuation: {total} total");
+            for sled in vmms_stopped_by_sled {
+                println!(
+                    "      sled_id={} n_stopped={}",
+                    sled.sled_id,
+                    sled.vmm_ids.len()
+                );
+            }
+            if !vmms_failed_to_stop_by_sled.is_empty() {
+                let total_failed: usize = vmms_failed_to_stop_by_sled
+                    .iter()
+                    .map(|s| s.vmm_ids.len())
+                    .sum();
+                println!("    VMMs that failed to stop: {total_failed} total");
+                for sled in vmms_failed_to_stop_by_sled {
+                    println!(
+                        "      sled_id={} n_failed_to_stop={}",
+                        sled.sled_id,
+                        sled.vmm_ids.len()
+                    );
+                    for vmm in sled.vmm_ids {
+                        println!("        vmm_id={}", vmm);
+                    }
+                }
+            }
+
+            if !error_messages.is_empty() {
+                let total_err = error_messages.len();
+                println!("    Error messages: {total_err} total");
+                for error in error_messages {
+                    println!("      {error}");
+                }
+            }
+        }
+    };
+}
+
 fn print_task_blueprint_planner(details: &serde_json::Value) {
     let status =
         match serde_json::from_value::<BlueprintPlannerStatus>(details.clone())

dev-tools/omdb/tests/env.out

@@ -271,6 +271,10 @@ task: "vmm_mark_stop_for_update"
     marks VMMs on evacuating sleds as needing to be stopped for an update
 
 
+task: "vmm_stop_for_update"
+    stops VMMs that are marked to be stopped for a sled update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -547,6 +551,10 @@ task: "vmm_mark_stop_for_update"
     marks VMMs on evacuating sleds as needing to be stopped for an update
 
 
+task: "vmm_stop_for_update"
+    stops VMMs that are marked to be stopped for a sled update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -810,6 +818,10 @@ task: "vmm_mark_stop_for_update"
     marks VMMs on evacuating sleds as needing to be stopped for an update
 
 
+task: "vmm_stop_for_update"
+    stops VMMs that are marked to be stopped for a sled update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 

dev-tools/omdb/tests/successes.out

@@ -601,6 +601,10 @@ task: "vmm_mark_stop_for_update"
     marks VMMs on evacuating sleds as needing to be stopped for an update
 
 
+task: "vmm_stop_for_update"
+    stops VMMs that are marked to be stopped for a sled update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -1209,6 +1213,12 @@ task: "vmm_mark_stop_for_update"
     batch size:                              1000
     VMMs marked to be stopped for an update: 0
 
+task: "vmm_stop_for_update"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    VMMs stopped for sled evacuation: 0 total
+
 task: "vpc_route_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1946,6 +1956,12 @@ task: "vmm_mark_stop_for_update"
     batch size:                              1000
     VMMs marked to be stopped for an update: 0
 
+task: "vmm_stop_for_update"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    VMMs stopped for sled evacuation: 0 total
+
 task: "vpc_route_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

nexus-config/src/nexus_config.rs

@@ -457,6 +457,9 @@ pub struct BackgroundTaskConfig {
     pub v2p_mapping_propagation: V2PMappingPropagationConfig,
     /// configuration for abandoned VMM reaper task
     pub abandoned_vmm_reaper: AbandonedVmmReaperConfig,
+    /// configuration for the task that stops VMMs marked to be stopped for an
+    /// update
+    pub vmm_stop_for_update: VmmStopForUpdateConfig,
     /// configuration for saga recovery task
     pub saga_recovery: SagaRecoveryConfig,
     /// configuration for lookup region port task
@@ -825,6 +828,14 @@ pub struct AbandonedVmmReaperConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct VmmStopForUpdateConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct SagaRecoveryConfig {
@@ -1387,6 +1398,7 @@ mod test {
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60
+            vmm_stop_for_update.period_secs = 300
             saga_recovery.period_secs = 60
             lookup_region_port.period_secs = 60
             region_snapshot_replacement_start.period_secs = 30
@@ -1627,6 +1639,9 @@ mod test {
                         abandoned_vmm_reaper: AbandonedVmmReaperConfig {
                             period_secs: Duration::from_secs(60),
                         },
+                        vmm_stop_for_update: VmmStopForUpdateConfig {
+                            period_secs: Duration::from_secs(300),
+                        },
                         saga_recovery: SagaRecoveryConfig {
                             period_secs: Duration::from_secs(60),
                         },
@@ -1803,6 +1818,7 @@ mod test {
             service_firewall_propagation.period_secs = 300
             v2p_mapping_propagation.period_secs = 30
             abandoned_vmm_reaper.period_secs = 60
+            vmm_stop_for_update.period_secs = 300
             saga_recovery.period_secs = 60
             lookup_region_port.period_secs = 60
             region_snapshot_replacement_start.period_secs = 30

nexus/background-task-interface/src/init.rs

@@ -36,6 +36,7 @@ pub struct BackgroundTasks {
     pub task_instance_reincarnation: Activator,
     pub task_service_firewall_propagation: Activator,
     pub task_abandoned_vmm_reaper: Activator,
+    pub task_vmm_stop_for_update: Activator,
     pub task_audit_log_cleanup: Activator,
     pub task_audit_log_timeout_incomplete: Activator,
     pub task_vpc_route_manager: Activator,

nexus/db-queries/src/db/datastore/vmm.rs

@@ -11,6 +11,7 @@ use crate::db::model;
 use crate::db::model::Vmm;
 use crate::db::model::VmmState as DbVmmState;
 use crate::db::pagination::paginated;
+use crate::db::pagination::paginated_multicolumn;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::db::update_and_check::UpdateAndQueryResult;
 use crate::db::update_and_check::UpdateStatus;
@@ -569,6 +570,26 @@ impl DataStore {
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
+
+    /// Lists VMMs that have been marked as needing to be stopped for a sled
+    /// update.
+    ///
+    /// The query performed by this function is paginated by the sled and
+    /// VMM UUIDs, in that order.
+    pub async fn vmm_list_marked_stop_for_update_by_sled_id(
+        &self,
+        opctx: &OpContext,
+        pagparams: &DataPageParams<'_, (Uuid, Uuid)>,
+    ) -> ListResultVec<Vmm> {
+        paginated_multicolumn(dsl::vmm, (dsl::sled_id, dsl::id), pagparams)
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::stop_for_update_disposition_generation.is_not_null())
+            .filter(dsl::state.eq_any(DbVmmState::SHOULD_STOP_FOR_EVACUATION))
+            .select(Vmm::as_select())
+            .load_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
 }
 
 #[cfg(test)]
@@ -592,6 +613,7 @@ mod tests {
     use omicron_uuid_kinds::InstanceUuid;
     use omicron_uuid_kinds::SledUuid;
     use sled_agent_types::instance::MigrationState;
+    use std::collections::BTreeSet;
     use std::collections::HashMap;
 
     #[tokio::test]
@@ -1325,6 +1347,107 @@ mod tests {
         logctx.cleanup_successful();
     }
 
+    #[tokio::test]
+    async fn test_vmm_list_marked_stop_for_update_by_sled_id() {
+        // Setup
+        let logctx =
+            dev::test_setup_log("test_vmm_list_marked_stop_for_update");
+        let db = TestDatabase::new_with_datastore(&logctx.log).await;
+        let (opctx, datastore) = (db.opctx(), db.datastore());
+        let conn = datastore.pool_connection_for_tests().await.unwrap();
+
+        // The states from which a VMM can still be stopped for an update. Only
+        // marked VMMs in these states should be listed by
+        // `vmm_list_marked_stop_for_update`.
+        let is_stoppable = |state: DbVmmState| {
+            matches!(
+                state,
+                DbVmmState::Creating
+                    | DbVmmState::Starting
+                    | DbVmmState::Running
+                    | DbVmmState::Rebooting
+            )
+        };
+
+        // Insert 2 VMMs (one marked to stop, one not) in each state per sled
+        // (2 sleds only).
+        let generation = UpdateDispositionGeneration::from(1);
+        let sled_a = SledUuid::new_v4();
+        let sled_b = SledUuid::new_v4();
+        let mut vmms = Vec::new();
+        for sled_id in [sled_a, sled_b] {
+            for &state in DbVmmState::ALL_STATES {
+                for marker in [Some(generation), None] {
+                    let failure_reason = (state == DbVmmState::Failed)
+                        .then_some(db::model::VmmFailureReason::FromSledAgent);
+                    let vmm = datastore
+                        .vmm_insert(
+                            &opctx,
+                            Vmm {
+                                id: Uuid::new_v4(),
+                                time_created: Utc::now(),
+                                time_deleted: None,
+                                instance_id: Uuid::new_v4(),
+                                sled_id: sled_id.into(),
+                                propolis_ip: "10.1.9.32".parse().unwrap(),
+                                propolis_port: 420.into(),
+                                cpu_platform: VmmCpuPlatform::SledDefault,
+                                time_state_updated: Utc::now(),
+                                generation: Generation::new(),
+                                state,
+                                failure_reason,
+                                stop_for_update_disposition_generation: marker
+                                    .map(Into::into),
+                            },
+                        )
+                        .await
+                        .expect("VMM should be inserted successfully");
+                    vmms.push(vmm);
+                }
+            }
+        }
+
+        // There should be 40 VMM rows. 2 VMMs (one marked, one not) per state
+        // per sled
+        let row_count: i64 = dsl::vmm
+            // Add a filter to ensure there is no full table scan
+            .filter(dsl::id.ne(Uuid::nil()))
+            .count()
+            .get_result_async(&*conn)
+            .await
+            .expect("VMMs should be counted");
+        assert_eq!(row_count, 40);
+
+        // Check there are 8 stoppable VMMs (4 per sled)
+        let expected_ids: BTreeSet<Uuid> = vmms
+            .iter()
+            .filter(|vmm| {
+                vmm.stop_for_update_disposition_generation.is_some()
+                    && is_stoppable(vmm.state)
+            })
+            .map(|vmm| vmm.id)
+            .collect();
+        assert_eq!(expected_ids.len(), 8);
+
+        // Check the retrieved ids for stoppable VMMs are the same as the
+        // expected ones
+        let listed = datastore
+            .vmm_list_marked_stop_for_update_by_sled_id(
+                &opctx,
+                &DataPageParams::max_page(),
+            )
+            .await
+            .expect("marked VMMs should be listed");
+        assert_eq!(listed.len(), expected_ids.len());
+        let listed_ids: BTreeSet<Uuid> =
+            listed.iter().map(|vmm| vmm.id).collect();
+        assert_eq!(listed_ids, expected_ids);
+
+        // Clean up.
+        db.terminate().await;
+        logctx.cleanup_successful();
+    }
+
     async fn mark_failed_no_such_instance(
         opctx: &OpContext,
         datastore: &DataStore,

nexus/examples/config-second.toml

@@ -162,6 +162,7 @@ instance_reincarnation.period_secs = 60
 service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+vmm_stop_for_update.period_secs = 300
 saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 region_snapshot_replacement_start.period_secs = 30

nexus/examples/config.toml

@@ -153,6 +153,7 @@ instance_reincarnation.period_secs = 60
 service_firewall_propagation.period_secs = 300
 v2p_mapping_propagation.period_secs = 30
 abandoned_vmm_reaper.period_secs = 60
+vmm_stop_for_update.period_secs = 300
 saga_recovery.period_secs = 600
 lookup_region_port.period_secs = 60
 region_snapshot_replacement_start.period_secs = 30