[live-update] adapt instance update saga to account for VMMs stopped by reconfigurator

nexus/db-model/src/instance_state.rs

@@ -30,6 +30,7 @@ impl_enum_type!(
     Vmm => b"vmm"
     Failed => b"failed"
     Destroyed => b"destroyed"
+    SledEvacuating => b"sled_evacuating"
 );
 
 impl InstanceState {
@@ -49,6 +50,7 @@ impl InstanceState {
             InstanceState::Vmm => "VMM",
             InstanceState::Failed => "failed",
             InstanceState::Destroyed => "destroyed",
+            InstanceState::SledEvacuating => "sled_evacuating",
         }
     }
 
@@ -71,6 +73,7 @@ impl From<InstanceState> for instance::InstanceState {
             InstanceState::Vmm => Output::Running,
             InstanceState::Failed => Output::Failed,
             InstanceState::Destroyed => Output::Destroyed,
+            InstanceState::SledEvacuating => Output::Rebooting,
         }
     }
 }

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(307, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(308, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ pub static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(308, "instance-state-sled-evacuating"),
         KnownVersion::new(307, "add-rack-id-to-inv-sled-agent"),
         KnownVersion::new(306, "inv-service-processor-rack-id"),
         KnownVersion::new(305, "lookup-sled-by-bp-availability"),

nexus/db-model/src/vmm_state.rs

@@ -350,6 +350,11 @@ impl<'s> InstanceStateComputer<'s> {
             // - An instance with no VMM is always "stopped" (as long as it's
             //   not "starting" etc.)
             (InstanceState::NoVmm, None) => instance::InstanceState::Stopped,
+            // - An instance with no VMM because it was in a sled that is being
+            //   evacuated is always "rebooting"
+            (InstanceState::SledEvacuating, None) => {
+                instance::InstanceState::Rebooting
+            }
             // - The instance should not be in `InstanceState::Vmm` if there is
             //   no active VMM record, this is probably a bug, but return
             //   `InstanceState::Stopped`, because that's basically true
@@ -370,6 +375,14 @@ impl<'s> InstanceStateComputer<'s> {
                 );
                 instance::InstanceState::Stopped
             }
+            (InstanceState::SledEvacuating, _vmm_state) => {
+                debug_assert_eq!(
+                    _vmm_state, None,
+                    "if the instance is in `InstanceState::SledEvacuating`, \
+                     there should be no VMM state"
+                );
+                instance::InstanceState::Rebooting
+            }
             // - If there's no VMM state, use the instance's state.
             (instance_state, None) => instance_state.into(),
             (instance_state, _vmm_state) => {

nexus/src/app/sagas/instance_common.rs

@@ -239,7 +239,9 @@ pub(super) async fn networking_resource_instance_state(
     // - starting: see below.
     match (found_instance_state, found_vmm_state) {
         // If there's no VMM, the instance is definitely not on any sled.
-        (InstanceState::NoVmm, _) | (_, Some(VmmState::SagaUnwound)) => {
+        (InstanceState::NoVmm, _)
+        | (InstanceState::SledEvacuating, _)
+        | (_, Some(VmmState::SagaUnwound)) => {
             propolis_and_sled_id = None;
         }
 

nexus/src/app/sagas/instance_update/mod.rs

@@ -545,6 +545,7 @@ impl UpdatesRequired {
 
         let mut update_required = false;
         let mut active_vmm_failed = false;
+        let mut active_vmm_update_disposition_gen = None;
         let mut network_config = None;
         let mut update_sled_reservations_for_migration_success = None;
 
@@ -564,23 +565,31 @@ impl UpdatesRequired {
                     // instance's new state to `Failed` rather than to `NoVmm`.
                     if active_vmm.state == VmmState::Failed {
                         active_vmm_failed = true;
-                    } else if snapshot.instance.intended_state
-                        == InstanceIntendedState::Running
-                        && snapshot.migration.is_none()
-                    {
-                        // Did the active VMM shut itself down, when it was
-                        // intended to be running *and* we were not migrating
-                        // out? If so, update the instance's intended state to
-                        // indicate that it stopped itself.
-                        new_intent = Some(InstanceIntendedState::GuestShutdown);
-                        info!(
-                            log,
-                            "instance update (active VMM destroyed): guest shut \
-                             itself down";
-                            "instance_id" => %instance_id,
-                            "propolis_id" => %active_vmm.id,
-                        );
                     }
+                    // If the active VMM is marked as stopped for an update, its
+                    // sled is being evacuated. Move the instance's new state to
+                    // `SledEvacuating` rather than to `NoVmm`.
+                    else if let Some(generation) = active_vmm
+                    .stop_for_update_disposition_generation {
+                        active_vmm_update_disposition_gen = Some(generation);
+
+                    } else if snapshot.instance.intended_state
+                    == InstanceIntendedState::Running
+                    && snapshot.migration.is_none()
+                {
+                    // Did the active VMM shut itself down, when it was
+                    // intended to be running *and* we were not migrating
+                    // out? If so, update the instance's intended state to
+                    // indicate that it stopped itself.
+                    new_intent = Some(InstanceIntendedState::GuestShutdown);
+                    info!(
+                        log,
+                        "instance update (active VMM destroyed): guest shut \
+                         itself down";
+                        "instance_id" => %instance_id,
+                        "propolis_id" => %active_vmm.id,
+                    );
+                }
 
                     Some(id)
                 } else {
@@ -740,6 +749,16 @@ impl UpdatesRequired {
             // We no longer have a VMM.
             new_runtime.nexus_state = if active_vmm_failed {
                 InstanceState::Failed
+            } else if let Some(g) = active_vmm_update_disposition_gen {
+                debug!(
+                    log,
+                    "instance update: VMM was stopped due to its sled being \
+                     evacuated for an update. Setting new runtime instance \
+                     state to SledEvacuating.";
+                    "instance_id" => %instance_id,
+                    "update_disposition_generation" => ?g
+                );
+                InstanceState::SledEvacuating
             } else {
                 InstanceState::NoVmm
             };

schema/crdb/dbinit.sql

@@ -1315,7 +1315,13 @@ CREATE TYPE IF NOT EXISTS omicron.public.instance_state_v2 AS ENUM (
     'failed',
 
     /* The instance has been destroyed. */
-    'destroyed'
+    'destroyed',
+
+    /* 
+     * The instance has no active VMM due to a sled being evacutated.
+     * Corresponds to the "rebooting" external state.
+     */
+    'sled_evacuating'
 );
 
 CREATE TYPE IF NOT EXISTS omicron.public.vmm_state AS ENUM (
@@ -9567,7 +9573,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    (TRUE, NOW(), NOW(), '307.0.0', NULL)
+    (TRUE, NOW(), NOW(), '308.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;

schema/crdb/instance-state-sled-evacuating/up1.sql

@@ -0,0 +1,7 @@
+ALTER TYPE
+  omicron.public.instance_state_v2
+ADD VALUE IF NOT EXISTS
+  'sled_evacuating'
+AFTER
+  'destroyed'
+;