diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 25135956b42..7f46629ebc5 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -31,6 +31,7 @@ use nexus_types_versions::v2026_01_08_00; use nexus_types_versions::v2026_01_15_00; use nexus_types_versions::v2026_01_16_00; use nexus_types_versions::v2026_01_16_01; +use nexus_types_versions::v2026_01_21_00; use nexus_types_versions::v2026_01_22_00; use nexus_types_versions::v2026_01_30_01; use nexus_types_versions::v2026_01_31_00; @@ -88,6 +89,7 @@ api_versions!([ // | date-based version should be at the top of the list. // v // (next_yyyy_mm_dd_nn, IDENT), + (2026_10_29_00, SLED_ADD_PRECHECK_OVERRIDES), (2026_10_21_00, BYTE_WRAPPER_0_2_BUMP), (2026_10_15_00, SYSTEM_CERTIFICATES), (2026_10_13_00, ARTIFACT_HASH_SCHEMA), @@ -7693,7 +7695,7 @@ pub trait NexusExternalApi { method = POST, path = "/v1/system/hardware/racks/{rack_id}/membership/add", tags = ["experimental"], - versions = VERSION_TRUST_QUORUM_ADD_SLEDS_AND_GET_LATEST_CONFIG.. + versions = VERSION_SLED_ADD_PRECHECK_OVERRIDES.. }] async fn rack_membership_add_sleds( rqctx: RequestContext, @@ -7701,6 +7703,30 @@ pub trait NexusExternalApi { req: TypedBody, ) -> Result, HttpError>; + /// Add new sleds to rack membership + #[endpoint { + operation_id = "rack_membership_add_sleds", + method = POST, + path = "/v1/system/hardware/racks/{rack_id}/membership/add", + tags = ["experimental"], + versions = VERSION_TRUST_QUORUM_ADD_SLEDS_AND_GET_LATEST_CONFIG..VERSION_SLED_ADD_PRECHECK_OVERRIDES, + }] + async fn rack_membership_add_sleds_v2026_01_21_00( + rqctx: RequestContext, + path_params: Path, + req: TypedBody, + ) -> Result< + HttpResponseOk, + HttpError, + > { + Self::rack_membership_add_sleds( + rqctx, + path_params, + req.map(latest::rack::RackMembershipAddSledsRequest::from), + ) + .await + } + /// Abort the latest rack membership change /// /// This operation is synchronous. Upon returning from the API call, a diff --git a/nexus/src/app/sled_add_checks.rs b/nexus/src/app/sled_add_checks.rs index dc99b315d1c..25e57d54061 100644 --- a/nexus/src/app/sled_add_checks.rs +++ b/nexus/src/app/sled_add_checks.rs @@ -13,6 +13,7 @@ use nexus_reconfigurator_planning::mgs_updates::DeployedSledVersionsError; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintArtifactVersion; use nexus_types::deployment::MgsUpdateComponent; +use nexus_types::external_api::rack::SledAddPrecheckOverrides; use nexus_types::inventory::NotInInventory; use nexus_types::inventory::SpType; use nexus_types::tuf_repo::TufRepoDescription; @@ -20,18 +21,26 @@ use omicron_common::api::external::Error; use omicron_generation_kinds::TargetReleaseGeneration; use omicron_uuid_kinds::SledUuid; use sled_hardware_types::BaseboardId; +use slog::Logger; use slog::info; +use slog::warn; use slog_error_chain::InlineErrorChain; use std::fmt::Display; impl super::Nexus { /// Checks that both the system and the sleds are ready for these sleds to /// be added. + /// + /// `overrides` describes checks that the operator asked to skip. We still + /// run those checks when we can, but we log any problem they find instead + /// of returning it. pub(super) async fn sled_check_versions( &self, opctx: &OpContext, sleds: impl IntoIterator, + overrides: &SledAddPrecheckOverrides, ) -> Result<(), Error> { + let log = opctx.log.new(o!("overrides" => format!("{overrides:?}"))); let target_release = self.datastore().target_release_get_current(opctx).await?; let release_source = @@ -50,7 +59,7 @@ impl super::Nexus { // adopted self-service update. In these cases, it's not // obvious what versions of software we would check against. info!( - opctx.log, + &log, "skipping software checks for sled add: \ no target release has been set" ); @@ -66,12 +75,35 @@ impl super::Nexus { .into(); let (_, blueprint) = self.datastore().blueprint_target_get_current_full(opctx).await?; - check_system_settled(&blueprint, target_release.generation(), &repo) - .map_err(SledAddPrecheckError::from)?; + + // Each call to `apply_override` names the single field of `overrides` + // that applies to every error the check can return. + let SledAddPrecheckOverrides { + allow_without_settled_target_release, + allow_unverified_sled_software, + } = *overrides; + + apply_override( + &log, + allow_without_settled_target_release, + check_system_settled( + &blueprint, + target_release.generation(), + &repo, + ), + )?; let Some(inventory) = self.datastore().inventory_get_latest_collection(opctx).await? else { + if allow_unverified_sled_software { + warn!( + log, + "skipping software checks for sled add: no inventory \ + collection found, and the operator overrode these checks" + ); + return Ok(()); + } return Err(Error::internal_error("no inventory collection found")); }; @@ -86,6 +118,7 @@ impl super::Nexus { }) } Err(DeployedSledVersionsError::NotASled(sp_type)) => { + // This is not overridable. Return immediately. return Err(SledAddPrecheckError::NotASled { board: sled.clone(), sp_type, @@ -93,12 +126,37 @@ impl super::Nexus { .into()); } }; - report.map_err(SledAddPrecheckError::from)?; + apply_override(&log, allow_unverified_sled_software, report)?; } Ok(()) } } +/// Returns the error from a failed check unless `allowed` is set, in which +/// case the error is logged instead +fn apply_override( + log: &Logger, + allowed: bool, + result: Result<(), E>, +) -> Result<(), SledAddPrecheckError> +where + E: std::error::Error + Into, +{ + let Err(error) = result else { + return Ok(()); + }; + if !allowed { + return Err(error.into()); + } + warn!( + log, + "sled add precheck failed, but proceeding because the operator \ + overrode it"; + InlineErrorChain::new(&error), + ); + Ok(()) +} + /// Checks that the system has settled on the target release described by /// `repo` /// @@ -174,6 +232,9 @@ enum SledAddPrecheckError { } /// Reasons that the system has not settled on its target release +/// +/// The operator can override all of these with +/// `allow_without_settled_target_release`. #[derive(Debug, thiserror::Error)] enum SystemNotSettledError { #[error( @@ -206,6 +267,9 @@ enum SystemNotSettledError { /// Reasons that a sled's software could not be confirmed to match the target /// release +/// +/// The operator can override all of these with +/// `allow_unverified_sled_software`. #[derive(Debug, thiserror::Error)] enum SledSoftwareError { #[error("the software on sled {sled} could not be checked")] @@ -321,6 +385,7 @@ mod tests { use nexus_reconfigurator_planning::mgs_updates::RotCheckError; use nexus_types::inventory::CabooseVersionError; use nexus_types::inventory::CabooseWhich; + use omicron_test_utils::dev::test_setup_log; use sled_hardware_types::GIMLET_SLED_MODEL; use std::collections::BTreeMap; use tufaceous_artifact::ArtifactHash; @@ -465,4 +530,42 @@ mod tests { }] ); } + + #[test] + fn test_apply_override() { + let logctx = test_setup_log("test_apply_override"); + let log = &logctx.log; + let mismatch = || SledSoftwareError::Mismatch { + sled: test_sled(), + target_release_version: "17.0.0".parse().expect("valid version"), + problems: vec![SledSoftwareProblem::WrongImage { + component: MgsUpdateComponent::Sp, + found: "1.0.0".to_string(), + expected: "2.0.0".to_string(), + }], + }; + + // A check that passed is never an error, whether or not it was + // overridden. + apply_override::(log, false, Ok(())) + .expect("check passed"); + apply_override::(log, true, Ok(())) + .expect("check passed"); + + // A check that failed is an error unless it was overridden. + let error = apply_override(log, false, Err(mismatch())) + .expect_err("not overridden"); + assert!( + matches!( + error, + SledAddPrecheckError::SledSoftware( + SledSoftwareError::Mismatch { .. } + ) + ), + "unexpected error: {error:?}" + ); + apply_override(log, true, Err(mismatch())).expect("overridden"); + + logctx.cleanup_successful(); + } } diff --git a/nexus/src/app/trust_quorum.rs b/nexus/src/app/trust_quorum.rs index 2a4af58acc4..faff7216805 100644 --- a/nexus/src/app/trust_quorum.rs +++ b/nexus/src/app/trust_quorum.rs @@ -6,6 +6,7 @@ use nexus_auth::authz; use nexus_auth::context::OpContext; +use nexus_types::external_api::rack::SledAddPrecheckOverrides; use nexus_types::trust_quorum::{ IsLrtqUpgrade, ProposedTrustQuorumConfig, TrustQuorumConfig, }; @@ -28,6 +29,7 @@ impl super::Nexus { opctx: &OpContext, authz_tq: authz::TrustQuorumConfig, new_sleds: BTreeSet, + override_prechecks: &SledAddPrecheckOverrides, ) -> Result { let rack_id = authz_tq.rack().id(); let (latest_committed_config, latest_epoch) = self @@ -36,7 +38,7 @@ impl super::Nexus { // Check the new sleds' software before allowing them to be added to // the system. - self.sled_check_versions(opctx, &new_sleds).await?; + self.sled_check_versions(opctx, &new_sleds, override_prechecks).await?; let new_epoch = latest_epoch.next(); let proposed = self diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 8aa27083279..2dec7644f0e 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6616,13 +6616,17 @@ impl NexusExternalApi for NexusExternalApiImpl { req: TypedBody, ) -> Result, HttpError> { audit_and_time(&rqctx, |opctx, nexus| async move { - let req = req.into_inner(); + let rack::RackMembershipAddSledsRequest { + sled_ids, + override_prechecks, + } = req.into_inner(); let rack_id = path_params.into_inner().rack_id; let authz_tq = authz::TrustQuorumConfig::for_rack_id( RackUuid::from_untyped_uuid(rack_id), ); - let status = - nexus.tq_add_sleds(&opctx, authz_tq, req.sled_ids).await?; + let status = nexus + .tq_add_sleds(&opctx, authz_tq, sled_ids, &override_prechecks) + .await?; Ok(HttpResponseOk(status.into())) }) .await diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 2eb034232ca..333f17604a1 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -110,6 +110,7 @@ pub static DEMO_RACK_ADD_SLEDS_REQUEST: LazyLock< serial_number: "demo-serial".to_string(), part_number: "demo-part".to_string(), }]), + override_prechecks: rack::SledAddPrecheckOverrides::no_overrides_enabled(), }); pub static DEMO_SLED_PROVISION_POLICY: LazyLock< diff --git a/nexus/types/versions/src/latest.rs b/nexus/types/versions/src/latest.rs index 7c551cdad77..399d40d16cb 100644 --- a/nexus/types/versions/src/latest.rs +++ b/nexus/types/versions/src/latest.rs @@ -517,12 +517,14 @@ pub mod physical_disk { pub mod rack { pub use crate::v2025_11_20_00::rack::Rack; - pub use crate::v2026_01_21_00::rack::RackMembershipAddSledsRequest; pub use crate::v2026_01_21_00::rack::RackMembershipChangeState; pub use crate::v2026_01_21_00::rack::RackMembershipConfigPathParams; pub use crate::v2026_01_21_00::rack::RackMembershipStatus; pub use crate::v2026_01_21_00::rack::RackMembershipVersion; pub use crate::v2026_01_21_00::rack::RackMembershipVersionParam; + + pub use crate::v2026_10_29_00::rack::RackMembershipAddSledsRequest; + pub use crate::v2026_10_29_00::rack::SledAddPrecheckOverrides; } pub mod subnet_pool { diff --git a/nexus/types/versions/src/lib.rs b/nexus/types/versions/src/lib.rs index fbf66da0f3c..2816bff9d4f 100644 --- a/nexus/types/versions/src/lib.rs +++ b/nexus/types/versions/src/lib.rs @@ -111,3 +111,5 @@ pub mod v2026_09_11_00; pub mod v2026_09_15_00; #[path = "artifact_hash_schema/mod.rs"] pub mod v2026_10_13_00; +#[path = "sled_add_precheck_overrides/mod.rs"] +pub mod v2026_10_29_00; diff --git a/nexus/types/versions/src/sled_add_precheck_overrides/mod.rs b/nexus/types/versions/src/sled_add_precheck_overrides/mod.rs new file mode 100644 index 00000000000..94fc64efef4 --- /dev/null +++ b/nexus/types/versions/src/sled_add_precheck_overrides/mod.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `SLED_ADD_PRECHECK_OVERRIDES` of the Nexus external API. +//! +//! Adds `override_prechecks` to the request to add sleds to a rack. + +pub mod rack; diff --git a/nexus/types/versions/src/sled_add_precheck_overrides/rack.rs b/nexus/types/versions/src/sled_add_precheck_overrides/rack.rs new file mode 100644 index 00000000000..ec6ca31f0a6 --- /dev/null +++ b/nexus/types/versions/src/sled_add_precheck_overrides/rack.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Rack membership types for version SLED_ADD_PRECHECK_OVERRIDES. + +use crate::v2026_01_21_00; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sled_hardware_types::BaseboardId; +use std::collections::BTreeSet; + +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct RackMembershipAddSledsRequest { + pub sled_ids: BTreeSet, + /// Optional overrides for various safety checks + /// + /// Overriding checks is dangerous and should only be done at the direction + /// of Oxide support. + #[serde(default)] + pub override_prechecks: SledAddPrecheckOverrides, +} + +impl From + for RackMembershipAddSledsRequest +{ + fn from(old: v2026_01_21_00::rack::RackMembershipAddSledsRequest) -> Self { + let v2026_01_21_00::rack::RackMembershipAddSledsRequest { sled_ids } = + old; + RackMembershipAddSledsRequest { + sled_ids, + override_prechecks: SledAddPrecheckOverrides::default(), + } + } +} + +/// Instructs the system to override safety checks when adding a sled +#[derive( + Clone, + Debug, + Default, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct SledAddPrecheckOverrides { + /// Add the requested sleds even if the system has not settled on its + /// current target release + /// + /// This allows sleds to be added when the system is in the middle of a + /// software update or if it has recently been MUPdated and that MUPdate + /// hasn't been resolved. This is not generally supported because in order + /// for adding a sled to work, the versions of software on that sled must + /// match the rest of the control plane, but that's not well-defined in + /// these two cases. + /// + /// This override is provided for working around bugs and only in emergency + /// situations. It should not generally be used. + #[serde(default)] + pub allow_without_settled_target_release: bool, + + /// Add the requested sleds even if their software does not match the + /// current target release + /// + /// This allows sleds to be added even when their SP, RoT, RoT bootloader, + /// or host OS phase 1 images cannot be confirmed to match those expected by + /// the control plane. Adding sleds with mismatched software is not + /// supported since the control plane needs compatible versions in order to + /// communicate with the sled. + /// + /// This override is provided for working around bugs and only in emergency + /// situations. It should not generally be used. + #[serde(default)] + pub allow_unverified_sled_software: bool, +} + +impl SledAddPrecheckOverrides { + pub fn no_overrides_enabled() -> Self { + Self { + allow_without_settled_target_release: false, + allow_unverified_sled_software: false, + } + } +} diff --git a/openapi/nexus/nexus-2026102100.0.0-b37141.json.gitstub b/openapi/nexus/nexus-2026102100.0.0-b37141.json.gitstub new file mode 100644 index 00000000000..704fc46179f --- /dev/null +++ b/openapi/nexus/nexus-2026102100.0.0-b37141.json.gitstub @@ -0,0 +1 @@ +393e660eeb05fbc1048401d655fbaa15694116c1:openapi/nexus/nexus-2026102100.0.0-b37141.json diff --git a/openapi/nexus/nexus-2026102100.0.0-b37141.json b/openapi/nexus/nexus-2026102900.0.0-2e88b6.json similarity index 99% rename from openapi/nexus/nexus-2026102100.0.0-b37141.json rename to openapi/nexus/nexus-2026102900.0.0-2e88b6.json index b550268bc25..4e484e91dd0 100644 --- a/openapi/nexus/nexus-2026102100.0.0-b37141.json +++ b/openapi/nexus/nexus-2026102900.0.0-2e88b6.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "2026102100.0.0" + "version": "2026102900.0.0" }, "paths": { "/device/auth": { @@ -26949,6 +26949,18 @@ "RackMembershipAddSledsRequest": { "type": "object", "properties": { + "override_prechecks": { + "description": "Optional overrides for various safety checks\n\nOverriding checks is dangerous and should only be done at the direction of Oxide support.", + "default": { + "allow_unverified_sled_software": false, + "allow_without_settled_target_release": false + }, + "allOf": [ + { + "$ref": "#/components/schemas/SledAddPrecheckOverrides" + } + ] + }, "sled_ids": { "type": "array", "items": { @@ -28469,6 +28481,22 @@ "usable_physical_ram" ] }, + "SledAddPrecheckOverrides": { + "description": "Instructs the system to override safety checks when adding a sled", + "type": "object", + "properties": { + "allow_unverified_sled_software": { + "description": "Add the requested sleds even if their software does not match the current target release\n\nThis allows sleds to be added even when their SP, RoT, RoT bootloader, or host OS phase 1 images cannot be confirmed to match those expected by the control plane. Adding sleds with mismatched software is not supported since the control plane needs compatible versions in order to communicate with the sled.\n\nThis override is provided for working around bugs and only in emergency situations. It should not generally be used.", + "default": false, + "type": "boolean" + }, + "allow_without_settled_target_release": { + "description": "Add the requested sleds even if the system has not settled on its current target release\n\nThis allows sleds to be added when the system is in the middle of a software update or if it has recently been MUPdated and that MUPdate hasn't been resolved. This is not generally supported because in order for adding a sled to work, the versions of software on that sled must match the rest of the control plane, but that's not well-defined in these two cases.\n\nThis override is provided for working around bugs and only in emergency situations. It should not generally be used.", + "default": false, + "type": "boolean" + } + } + }, "SledInstance": { "description": "An operator's view of an instance running on a given sled", "type": "object", diff --git a/openapi/nexus/nexus-latest.json b/openapi/nexus/nexus-latest.json index cc951f48fc0..633d9d1631e 120000 --- a/openapi/nexus/nexus-latest.json +++ b/openapi/nexus/nexus-latest.json @@ -1 +1 @@ -nexus-2026102100.0.0-b37141.json \ No newline at end of file +nexus-2026102900.0.0-2e88b6.json \ No newline at end of file