From d507f3e456fe34177576337382da2f9a5224dede Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 20 Oct 2026 19:00:03 +0000 Subject: [PATCH 1/5] Add sled-agent API to report multirack join service status We need a way to discover information about sleds in unadopted racks so that we can show them to operators. The easiest way to do this is to return that information from the multirack join service, which already has it. This provides a sled-agent API endpoint to do just that. It adds a separate watch channel for just the status we care about for the control plane and utilizes that via the `MultirackJoinAccessor`, which was formally the `MultirackJoinCompleter`. Now, the question is: How do we find the sled-agent running the multirack join service? That is going to be solved in a separate PR. The sled-agent running the join service will publish the new rack subnet to announce along with its own IP to ddmd. DDMD will announce this information which will make its way to the proto rack ddmd servers. Nexuses on the proto rack can then poll their local sled-agent to discover the location of the sled-agent running the multirack join service and ask it for this info. Nexus can also complete the multirack join via the same sled-agent when the user adopts the new rack into the cluster. See https://github.com/oxidecomputer/maghemite/issues/939 --- .../src/test_util/host_phase_2_test_state.rs | 8 ++ .../sled-agent-57.0.0-621afe.json.gitstub | 1 + ...afe.json => sled-agent-58.0.0-612c52.json} | 126 +++++++++++++++++- openapi/sled-agent/sled-agent-latest.json | 2 +- sled-agent/api/src/lib.rs | 17 +++ sled-agent/multirack-join/src/lib.rs | 65 ++++++++- sled-agent/src/bootstrap/mod.rs | 2 +- sled-agent/src/bootstrap/rack_ops.rs | 75 ++++++----- sled-agent/src/bootstrap/server.rs | 8 +- sled-agent/src/http_entrypoints.rs | 18 +++ sled-agent/src/server.rs | 4 +- sled-agent/src/sim/http_entrypoints.rs | 7 + sled-agent/src/sled_agent.rs | 16 ++- sled-agent/types/src/lib.rs | 1 + sled-agent/types/src/multirack.rs | 7 + .../add_multirack_join_service_status/mod.rs | 7 + .../multirack.rs | 53 ++++++++ sled-agent/types/versions/src/latest.rs | 6 + sled-agent/types/versions/src/lib.rs | 2 + 19 files changed, 381 insertions(+), 44 deletions(-) create mode 100644 openapi/sled-agent/sled-agent-57.0.0-621afe.json.gitstub rename openapi/sled-agent/{sled-agent-57.0.0-621afe.json => sled-agent-58.0.0-612c52.json} (99%) create mode 100644 sled-agent/types/src/multirack.rs create mode 100644 sled-agent/types/versions/src/add_multirack_join_service_status/mod.rs create mode 100644 sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs diff --git a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs index 8386deca2b5..de81f420b3b 100644 --- a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs +++ b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs @@ -254,6 +254,7 @@ mod api_impl { use sled_agent_types::inventory::OmicronSledUpdateDisposition; use sled_agent_types::inventory::SledCpuFamily; use sled_agent_types::inventory::SvcsEnabledNotOnlineResult; + use sled_agent_types::multirack::MultirackJoinServiceStatus; use sled_agent_types::probes::ProbeSet; use sled_agent_types::sled::AddSledRequest; use sled_agent_types::support_bundle::RangeRequestHeaders; @@ -1152,5 +1153,12 @@ mod api_impl { ) -> Result { unimplemented!() } + + async fn multirack_join_service_status( + _request_context: RequestContext, + ) -> Result, HttpError> + { + unimplemented!() + } } } diff --git a/openapi/sled-agent/sled-agent-57.0.0-621afe.json.gitstub b/openapi/sled-agent/sled-agent-57.0.0-621afe.json.gitstub new file mode 100644 index 00000000000..6b11000b08f --- /dev/null +++ b/openapi/sled-agent/sled-agent-57.0.0-621afe.json.gitstub @@ -0,0 +1 @@ +564c09295bfacfbe1b1223b9936e5190f1f80513:openapi/sled-agent/sled-agent-57.0.0-621afe.json diff --git a/openapi/sled-agent/sled-agent-57.0.0-621afe.json b/openapi/sled-agent/sled-agent-58.0.0-612c52.json similarity index 99% rename from openapi/sled-agent/sled-agent-57.0.0-621afe.json rename to openapi/sled-agent/sled-agent-58.0.0-612c52.json index fc977004c16..ba5a349fad7 100644 --- a/openapi/sled-agent/sled-agent-57.0.0-621afe.json +++ b/openapi/sled-agent/sled-agent-58.0.0-612c52.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "57.0.0" + "version": "58.0.0" }, "paths": { "/artifacts": { @@ -395,6 +395,31 @@ } } }, + "/multirack-join-service-status": { + "get": { + "summary": "Return the status of this sled's multirack join service.", + "description": "Returns a 404 if no multirack join service is running, or a 409 if the service is running but has not yet determined its status.", + "operationId": "multirack_join_service_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MultirackJoinServiceStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/network-bootstore-config": { "put": { "operationId": "write_network_bootstore_config", @@ -7190,6 +7215,44 @@ } ] }, + "MultirackJoinServiceStatus": { + "type": "object", + "properties": { + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Subnet" + }, + "sled_agents": { + "title": "BiHashMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/SledAgentInfo" + } + ], + "path": "iddqd::BiHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/SledAgentInfo" + }, + "uniqueItems": true + }, + "sled_id": { + "$ref": "#/components/schemas/SledUuid" + } + }, + "required": [ + "rack_id", + "rack_subnet", + "sled_agents", + "sled_id" + ] + }, "MupdateOverrideBootInventory": { "description": "Inventory representation of the MUPdate override on the boot disk.", "type": "object", @@ -9590,6 +9653,67 @@ "result" ] }, + "SledAgentInfo": { + "description": "Status information for a given sled agent", + "type": "object", + "properties": { + "address": { + "type": "string", + "format": "ipv6" + }, + "baseboard_id": { + "$ref": "#/components/schemas/BaseboardId" + }, + "sled_id": { + "$ref": "#/components/schemas/SledUuid" + }, + "start_state": { + "$ref": "#/components/schemas/SledAgentStartState" + } + }, + "required": [ + "address", + "baseboard_id", + "sled_id", + "start_state" + ] + }, + "SledAgentStartState": { + "oneOf": [ + { + "type": "object", + "properties": { + "last_error": { + "nullable": true, + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "in_progress" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "started" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, "SledConfigGeneration": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "x-rust-type": { diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index e1a1b052f39..ae073689ed8 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-57.0.0-621afe.json \ No newline at end of file +sled-agent-58.0.0-612c52.json \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index c2bc80db2df..f749a37e438 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -36,6 +36,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (58, ADD_MULTIRACK_JOIN_SERVICE_STATUS), (57, ADD_RACK_ID_TO_INVENTORY), (56, ADD_MULTIRACK_JOIN_COMPLETE), (55, ARTIFACT_HASH_SCHEMA), @@ -1743,4 +1744,20 @@ pub trait SledAgentApi { async fn multirack_join_complete( request_context: RequestContext, ) -> Result; + + /// Return the status of this sled's multirack join service. + /// + /// Returns a 404 if no multirack join service is running, or a 409 if the + /// service is running but has not yet determined its status. + #[endpoint { + method = GET, + path = "/multirack-join-service-status", + versions = VERSION_ADD_MULTIRACK_JOIN_SERVICE_STATUS.., + }] + async fn multirack_join_service_status( + request_context: RequestContext, + ) -> Result< + HttpResponseOk, + HttpError, + >; } diff --git a/sled-agent/multirack-join/src/lib.rs b/sled-agent/multirack-join/src/lib.rs index ae5b8350e8c..5bcd4d31114 100644 --- a/sled-agent/multirack-join/src/lib.rs +++ b/sled-agent/multirack-join/src/lib.rs @@ -19,13 +19,18 @@ use bootstrap_agent_lockstep_types::{ CommitState, MultirackJoinRequest, MultirackJoinServiceState, SledAgentInfo, SledAgentStartState, StartSledAgentsStatus, }; +use iddqd::BiHashMap; use nexus_types::trust_quorum::TrustQuorumConfig; -use omicron_common::address::BOOTSTRAP_AGENT_RACK_INIT_PORT; +use omicron_common::address::{ + BOOTSTRAP_AGENT_RACK_INIT_PORT, Ipv6Subnet, RACK_PREFIX_LENGTH, + get_sled_address, +}; use omicron_ledger::{self as ledger}; use omicron_uuid_kinds::RackUuid; use sled_agent_bootstrap_common::sprockets::SprocketsClient; use sled_agent_bootstrap_common::{RssContext, RunRssError}; use sled_agent_types::early_networking::EarlyNetworkConfigEnvelope; +use sled_agent_types::multirack; use sled_agent_types::rack_init::rack_init_bootstore_generation; use sled_agent_types::sled::{ StartSledAgentRequest, StartSledAgentRequestBody, @@ -160,6 +165,8 @@ pub struct MultirackJoinServiceHandle { tokio::task::JoinHandle>, pub input_tx: watch::Sender, pub output_rx: watch::Receiver, + pub status_rx: + watch::Receiver>, // Once we've initialized trust quorum, we no longer allow membership // changes. This allows helpful responses to user requests in case a @@ -180,6 +187,7 @@ impl MultirackJoinServiceHandle { let (input_tx, input_rx) = watch::channel(request); let state = MultirackJoinServiceState::Requested; let (output_tx, output_rx) = watch::channel(state.clone()); + let (status_tx, status_rx) = watch::channel(None); let membership_change_still_possible = Arc::new(AtomicBool::new(true)); let mcsp = membership_change_still_possible.clone(); let (rack_join_successful_tx, rack_join_successful_rx) = @@ -194,6 +202,7 @@ impl MultirackJoinServiceHandle { ctx, input_rx, output_tx, + status_tx, membership_change_still_possible: mcsp, rack_join_successful_rx, bootstore_generation, @@ -205,6 +214,7 @@ impl MultirackJoinServiceHandle { join_handle, input_tx, output_rx, + status_rx, membership_change_still_possible, rack_join_successful_tx, } @@ -217,6 +227,7 @@ struct MultirackJoinServiceTask { ctx: RssContext, input_rx: watch::Receiver, output_tx: watch::Sender, + status_tx: watch::Sender>, // Once we've initialized trust quorum, we no longer allow membership // changes. This allows helpful responses to user requests in case a @@ -367,11 +378,40 @@ impl MultirackJoinServiceTask { // `changed()` wouldn't fire. let req = self.input_rx.borrow().clone(); let tq_members = req.trust_quorum_peers.clone(); + let rack_subnet = Ipv6Subnet::::new( + req.rack_network_config.rack_subnet.addr(), + ); let status = StartSledAgentsStatus::assign_ids_and_subnets(req); self.output_tx.send_modify(|state| { *state = MultirackJoinServiceState::StartSledAgents(status.clone()) }); + let sled_agents: BiHashMap = status + .sleds + .iter() + .map(|info| multirack::SledAgentInfo { + baseboard_id: info.baseboard_id.clone(), + sled_id: info.sled_id, + address: *get_sled_address(info.sled_subnet).ip(), + start_state: multirack::SledAgentStartState::InProgress { + last_error: None, + }, + }) + .collect(); + + if let Some(info) = + status.sleds.get1(self.ctx.trust_quorum_handle.baseboard_id()) + { + self.status_tx.send_replace(Some( + multirack::MultirackJoinServiceStatus { + sled_id: info.sled_id, + rack_id, + rack_subnet, + sled_agents, + }, + )); + } + let mut bootstrap_ips: BTreeMap<_, _> = self .ctx .trust_quorum_handle @@ -433,6 +473,14 @@ impl MultirackJoinServiceTask { let mut info = status.sleds.get1_mut(&baseboard_id).unwrap(); info.start_state = SledAgentStartState::Started; }); + self.status_tx.send_modify(|status| { + if let Some(status) = status + && let Some(mut info) = + status.sled_agents.get1_mut(&baseboard_id) + { + info.start_state = multirack::SledAgentStartState::Started; + } + }); } Ok(()) @@ -472,6 +520,7 @@ impl MultirackJoinServiceTask { ); let output_tx = self.output_tx.clone(); + let status_tx = self.status_tx.clone(); let mut retry_count = 0; async move { // Retry indefinitely to start sled-agent. @@ -499,6 +548,20 @@ impl MultirackJoinServiceTask { info.start_state = SledAgentStartState::InProgress { last_error: Some(errstr) }; }); + status_tx.send_modify(|status| { + let errstr = + InlineErrorChain::new(&err).to_string(); + if let Some(status) = status + && let Some(mut sled) = status + .sled_agents + .get1_mut(&info.baseboard_id) + { + sled.start_state = + multirack::SledAgentStartState::InProgress { + last_error: Some(errstr), + }; + } + }); // Only log every minute, given the 6 second sleep below if retry_count % 10 == 0 { warn!(log, "Failed to start sled agent"; &err); diff --git a/sled-agent/src/bootstrap/mod.rs b/sled-agent/src/bootstrap/mod.rs index 6334963e6b5..df1c16bbb29 100644 --- a/sled-agent/src/bootstrap/mod.rs +++ b/sled-agent/src/bootstrap/mod.rs @@ -17,7 +17,7 @@ mod sprockets_server; pub(crate) mod trust_quorum_setup; pub(crate) use pre_server::BootstrapNetworking; -pub(crate) use rack_ops::MultirackJoinCompleter; +pub(crate) use rack_ops::MultirackJoinAccessor; pub use rack_ops::RssAccessError; use slog_error_chain::InlineErrorChain; diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index e70636be7b0..8dd7e407698 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -17,6 +17,7 @@ use sled_agent_multirack_join::MultirackJoinServiceError; use sled_agent_multirack_join::MultirackJoinServiceHandle; use sled_agent_rack_setup::RackInitializeRequestParams; use sled_agent_rack_setup::SetupServiceError; +use sled_agent_types::multirack::MultirackJoinServiceStatus; use slog_error_chain::InlineErrorChain; use std::mem; use std::sync::Arc; @@ -51,6 +52,8 @@ pub enum RssAccessError { RackSubnetChangeNotAllowed, #[error("Multirack join has not yet started")] MultirackJoinNotYetStarted, + #[error("Multirack join service status is not yet available")] + MultirackJoinStatusNotYetAvailable, #[error("Multirack join task exited unexpectedly")] MultirackJoinTaskExitedUnexpectedly, } @@ -81,6 +84,9 @@ impl RssAccessError { RssAccessError::MultirackJoinNotYetStarted => { "MultirackJoinNotYetStarted" } + RssAccessError::MultirackJoinStatusNotYetAvailable => { + "MultirackJoinStatusNotYetAvailable" + } RssAccessError::MultirackJoinTaskExitedUnexpectedly => { "MultirackJoinTaskExitedUnexpectedly" } @@ -119,7 +125,8 @@ impl From for HttpError { | RssAccessError::MultirackJoinCompleted | RssAccessError::MembershipChangeNoLongerAllowed | RssAccessError::RackSubnetChangeNotAllowed - | RssAccessError::MultirackJoinNotYetStarted => { + | RssAccessError::MultirackJoinNotYetStarted + | RssAccessError::MultirackJoinStatusNotYetAvailable => { HttpError::for_client_error( Some(err.error_code().to_string()), dropshot::ClientErrorStatusCode::CONFLICT, @@ -130,19 +137,12 @@ impl From for HttpError { } } -/// A mechanism for sled-agent to complete a multirack join request. -/// -/// When a sled-agent that performed a multirack join becomes visible on the -/// underlay network to an existing Nexus via inventory, it will receive a -/// request telling it to complete the multirack join. We provide a newtype -/// wrapper that allows triggering that completion without providing full access -/// to `RssAccess`, which could be used to start initialization or multirack -/// join requests outside of the official path. +/// A mechanism for sled-agent to access its local multirack join service task. #[derive(Clone)] -pub(crate) struct MultirackJoinCompleter(RssAccess); +pub(crate) struct MultirackJoinAccessor(RssAccess); -impl MultirackJoinCompleter { - pub fn new(rss_access: RssAccess) -> MultirackJoinCompleter { +impl MultirackJoinAccessor { + pub fn new(rss_access: RssAccess) -> MultirackJoinAccessor { Self(rss_access) } @@ -153,25 +153,12 @@ impl MultirackJoinCompleter { self.0.complete_multirack_join() } - /// Used to provide inventory with information about the particular - /// sled-agent that is performing the multirack join. This prevents Nexus - /// from having to guess or try all sled-agents. - // - // TODO: Remove this `unused` annotation when we add this functionality - // to inventory. - #[expect(unused)] - pub fn is_multirack_join_in_progress(&self) -> bool { - match &*self.0.status.lock().unwrap() { - RssStatus::MultirackJoinInProgress { .. } => true, - RssStatus::Uninitialized - | RssStatus::Initialized { .. } - | RssStatus::Initializing { .. } - | RssStatus::InitializationFailed { .. } - | RssStatus::InitializationPanicked { .. } - | RssStatus::MultirackJoinCompleted { .. } - | RssStatus::MultirackJoinFailed { .. } - | RssStatus::MultirackJoinPanicked { .. } => false, - } + /// Return the current status of the multirack join service if it is + /// running. + pub fn multirack_join_service_status( + &self, + ) -> Result, RssAccessError> { + self.0.multirack_join_service_status() } } @@ -420,6 +407,7 @@ impl RssAccess { join_handle, input_tx, output_rx, + status_rx, membership_change_still_possible, rack_join_successful_tx, } = MultirackJoinServiceHandle::spawn(ctx, request); @@ -427,6 +415,7 @@ impl RssAccess { id, input_tx, output_rx, + status_rx, membership_change_still_possible, rack_join_successful_tx: Some(rack_join_successful_tx), }; @@ -539,6 +528,29 @@ impl RssAccess { } } + pub(crate) fn multirack_join_service_status( + &self, + ) -> Result, RssAccessError> { + match &*self.status.lock().unwrap() { + RssStatus::MultirackJoinInProgress { status_rx, .. } => { + match status_rx.borrow().clone() { + Some(status) => Ok(Some(status)), + None => { + Err(RssAccessError::MultirackJoinStatusNotYetAvailable) + } + } + } + RssStatus::Uninitialized + | RssStatus::Initialized { .. } + | RssStatus::Initializing { .. } + | RssStatus::InitializationFailed { .. } + | RssStatus::InitializationPanicked { .. } + | RssStatus::MultirackJoinCompleted { .. } + | RssStatus::MultirackJoinFailed { .. } + | RssStatus::MultirackJoinPanicked { .. } => Ok(None), + } + } + pub(crate) fn get_multirack_join_state( &self, ) -> Result { @@ -611,6 +623,7 @@ enum RssStatus { id: MultirackJoinUuid, input_tx: watch::Sender, output_rx: watch::Receiver, + status_rx: watch::Receiver>, membership_change_still_possible: Arc, rack_join_successful_tx: Option>, }, diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index be3955207d7..4f5cc481ea1 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -10,7 +10,7 @@ use super::http_entrypoints_lockstep::BootstrapServerContext; use crate::bootstrap::maghemite; use crate::bootstrap::pre_server::BootstrapAgentStartup; use crate::bootstrap::pumpkind; -use crate::bootstrap::rack_ops::{MultirackJoinCompleter, RssAccess}; +use crate::bootstrap::rack_ops::{MultirackJoinAccessor, RssAccess}; use crate::bootstrap::secret_retriever::{ HardcodedSecretRetriever, TqOrLrtqSecretRetriever, }; @@ -205,7 +205,7 @@ impl Server { // Provide a mechanism for multirack join requests to be completed by sled-agent. let multirack_join_completer = - MultirackJoinCompleter::new(rss_access.clone()); + MultirackJoinAccessor::new(rss_access.clone()); // Start the bootstrap dropshot server. let bootstrap_context = BootstrapServerContext { @@ -370,7 +370,7 @@ async fn start_sled_agent( long_running_task_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, service_manager: ServiceManager, - multirack_join_completer: MultirackJoinCompleter, + multirack_join_completer: MultirackJoinAccessor, base_log: &Logger, log: &Logger, ) -> Result { @@ -520,7 +520,7 @@ struct Inner { service_manager: ServiceManager, _sprockets_server_handle: JoinHandle<()>, base_log: Logger, - multirack_join_completer: MultirackJoinCompleter, + multirack_join_completer: MultirackJoinAccessor, } impl Inner { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index aa962ea10b6..43476ac27f8 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -49,6 +49,7 @@ use sled_agent_types::instance::{ VmmPutStateResponse, VmmUnregisterResponse, VpcPathParam, }; use sled_agent_types::inventory::{Inventory, OmicronSledConfig}; +use sled_agent_types::multirack::MultirackJoinServiceStatus; use sled_agent_types::probes::ProbeSet; use sled_agent_types::rot::{ Attestation, CertificateChain, MeasurementLog, Nonce, RotPathParams, @@ -1853,4 +1854,21 @@ impl SledAgentApi for SledAgentImpl { }) .await } + + async fn multirack_join_service_status( + request_context: RequestContext, + ) -> Result, HttpError> { + let sa = request_context.context(); + sa.latencies() + .instrument_dropshot_handler(&request_context, async { + match sa.multirack_join_service_status()? { + Some(status) => Ok(HttpResponseOk(status)), + None => Err(HttpError::for_not_found( + None, + "No multirack join service running".to_string(), + )), + } + }) + .await + } } diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index d157cae5162..dd499b496f6 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -7,7 +7,7 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; -use crate::bootstrap::MultirackJoinCompleter; +use crate::bootstrap::MultirackJoinAccessor; use crate::long_running_tasks::LongRunningTaskHandles; use crate::nexus::make_nexus_client; use crate::services::ServiceManager; @@ -44,7 +44,7 @@ impl Server { long_running_tasks_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, services: ServiceManager, - multirack_join_completer: MultirackJoinCompleter, + multirack_join_completer: MultirackJoinAccessor, ) -> Result { info!(log, "setting up sled agent server"); diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index fed7625b3f8..4e208f7bb82 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -60,6 +60,7 @@ use sled_agent_types::instance::{ VmmPutStateResponse, VmmUnregisterResponse, VpcPathParam, }; use sled_agent_types::inventory::{Inventory, OmicronSledConfig}; +use sled_agent_types::multirack::MultirackJoinServiceStatus; use sled_agent_types::probes::ProbeSet; use sled_agent_types::rot::{ Attestation, CertificateChain, MeasurementLog, Nonce, RotPathParams, @@ -1166,6 +1167,12 @@ impl SledAgentApi for SledAgentSimImpl { ) -> Result { method_unimplemented() } + + async fn multirack_join_service_status( + _request_context: RequestContext, + ) -> Result, HttpError> { + method_unimplemented() + } } fn method_unimplemented() -> Result { diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 4c5257508b8..858afdb6625 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -5,7 +5,7 @@ //! Sled agent implementation use crate::artifact_store::{ArtifactStore, SledAgentArtifactStoreWrapper}; -use crate::bootstrap::MultirackJoinCompleter; +use crate::bootstrap::MultirackJoinAccessor; use crate::config::Config; use crate::hardware_monitor::HardwareMonitorHandle; use crate::instance_manager::InstanceManager; @@ -86,6 +86,7 @@ use sled_agent_types::instance::{ VmmUnregisterResponse, }; use sled_agent_types::inventory::{Inventory, OmicronSledConfig, SledRole}; +use sled_agent_types::multirack::MultirackJoinServiceStatus; use sled_agent_types::probes::ProbeCreate; use sled_agent_types::resolvable_files::{ PreparedOmicronZone, RemoveMupdateOverrideResult, ResolverStatus, @@ -528,7 +529,7 @@ struct SledAgentInner { measurements: Arc, // A mechanism to complete a multirack join - multirack_join_completer: MultirackJoinCompleter, + multirack_join_completer: MultirackJoinAccessor, } impl SledAgentInner { @@ -561,7 +562,7 @@ impl SledAgent { services: ServiceManager, long_running_task_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, - multirack_join_completer: MultirackJoinCompleter, + multirack_join_completer: MultirackJoinAccessor, ) -> Result { // Pass the "parent_log" to all subcomponents that want to set their own // "component" value. @@ -1725,6 +1726,15 @@ impl SledAgent { self.inner.multirack_join_completer.complete_multirack_join()?; Ok(()) } + + pub(crate) fn multirack_join_service_status( + &self, + ) -> Result, Error> { + Ok(self + .inner + .multirack_join_completer + .multirack_join_service_status()?) + } } #[derive(From, thiserror::Error, Debug, SlogInlineError)] diff --git a/sled-agent/types/src/lib.rs b/sled-agent/types/src/lib.rs index 2d87bbc1761..f6d2745df82 100644 --- a/sled-agent/types/src/lib.rs +++ b/sled-agent/types/src/lib.rs @@ -16,6 +16,7 @@ pub mod early_networking; pub mod firewall_rules; pub mod instance; pub mod inventory; +pub mod multirack; pub mod probes; pub mod rack_init; pub mod resolvable_files; diff --git a/sled-agent/types/src/multirack.rs b/sled-agent/types/src/multirack.rs new file mode 100644 index 00000000000..a31e573a749 --- /dev/null +++ b/sled-agent/types/src/multirack.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Status information about the multirack join service. + +pub use sled_agent_types_versions::latest::multirack::*; diff --git a/sled-agent/types/versions/src/add_multirack_join_service_status/mod.rs b/sled-agent/types/versions/src/add_multirack_join_service_status/mod.rs new file mode 100644 index 00000000000..ec6724fdbe7 --- /dev/null +++ b/sled-agent/types/versions/src/add_multirack_join_service_status/mod.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version that adds multirack join service status + +pub mod multirack; diff --git a/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs b/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs new file mode 100644 index 00000000000..62bd591fcda --- /dev/null +++ b/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs @@ -0,0 +1,53 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Status information about the multirack join service + +use std::net::Ipv6Addr; + +use iddqd::{BiHashItem, BiHashMap, bi_upcast}; +use omicron_common::address::{Ipv6Subnet, RACK_PREFIX_LENGTH}; +use omicron_uuid_kinds::{RackUuid, SledUuid}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sled_hardware_types::BaseboardId; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +pub struct MultirackJoinServiceStatus { + pub sled_id: SledUuid, + pub rack_id: RackUuid, + pub rack_subnet: Ipv6Subnet, + pub sled_agents: BiHashMap, +} + +/// Status information for a given sled agent +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct SledAgentInfo { + pub baseboard_id: BaseboardId, + pub sled_id: SledUuid, + pub address: Ipv6Addr, + pub start_state: SledAgentStartState, +} + +impl BiHashItem for SledAgentInfo { + type K1<'a> = &'a BaseboardId; + type K2<'a> = &'a SledUuid; + + fn key1(&self) -> Self::K1<'_> { + &self.baseboard_id + } + + fn key2(&self) -> Self::K2<'_> { + &self.sled_id + } + + bi_upcast!(); +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case", tag = "type")] +pub enum SledAgentStartState { + InProgress { last_error: Option }, + Started, +} diff --git a/sled-agent/types/versions/src/latest.rs b/sled-agent/types/versions/src/latest.rs index 63e9ba52ff7..a0d8da4fa67 100644 --- a/sled-agent/types/versions/src/latest.rs +++ b/sled-agent/types/versions/src/latest.rs @@ -231,6 +231,12 @@ pub mod inventory { pub use crate::impls::inventory::ZpoolHealthParseError; } +pub mod multirack { + pub use crate::v58::multirack::MultirackJoinServiceStatus; + pub use crate::v58::multirack::SledAgentInfo; + pub use crate::v58::multirack::SledAgentStartState; +} + pub mod probes { pub use crate::v10::probes::ExternalIp; pub use crate::v10::probes::IpKind; diff --git a/sled-agent/types/versions/src/lib.rs b/sled-agent/types/versions/src/lib.rs index 0566c7ac8ab..275a54a88da 100644 --- a/sled-agent/types/versions/src/lib.rs +++ b/sled-agent/types/versions/src/lib.rs @@ -117,6 +117,8 @@ pub mod v54; pub mod v55; #[path = "add_rack_id_to_inventory/mod.rs"] pub mod v57; +#[path = "add_multirack_join_service_status/mod.rs"] +pub mod v58; #[path = "add_probe_put_endpoint/mod.rs"] pub mod v6; #[path = "multicast_support/mod.rs"] From dde6f9f659b2812cffc632d80c0a708c5d6ef5e4 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 21 Oct 2026 20:39:21 +0000 Subject: [PATCH 2/5] review comments --- .../bootstrap-agent-lockstep-types/src/lib.rs | 97 ++++++++--- sled-agent/multirack-join/src/lib.rs | 132 +++++++------- .../bootstrap/http_entrypoints_lockstep.rs | 51 +++++- sled-agent/src/bootstrap/rack_ops.rs | 164 +++++++----------- sled-agent/src/http_entrypoints.rs | 56 +++++- sled-agent/src/sled_agent.rs | 6 +- .../multirack.rs | 1 + 7 files changed, 299 insertions(+), 208 deletions(-) diff --git a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs index a083438e5c4..df7f0d35840 100644 --- a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs @@ -19,6 +19,7 @@ use omicron_common::address::Ipv6Subnet; use omicron_common::address::RACK_PREFIX_LENGTH; use omicron_common::address::SLED_PREFIX_LENGTH; use omicron_common::address::get_64_subnet; +use omicron_common::address::get_sled_address; use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::Name; use omicron_common::api::external::UserId; @@ -30,6 +31,9 @@ use omicron_uuid_kinds::SledUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_agent_types::early_networking::RackNetworkConfig; +use sled_agent_types::multirack::{ + MultirackJoinServiceStatus, SledAgentStartState, +}; use sled_hardware_types::BaseboardId; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -446,15 +450,6 @@ pub struct CommitState { pub transient_errors: BTreeMap, } -// Information about the state of a sled agent start request -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "snake_case", tag = "type")] -pub enum SledAgentStartState { - InProgress { last_error: Option }, - Started, - Failed { reason: String }, -} - /// Status information for a given sled agent #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct SledAgentInfo { @@ -481,36 +476,92 @@ impl BiHashItem for SledAgentInfo { #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct StartSledAgentsStatus { + pub this_nodes_sled_id: SledUuid, + pub rack_id: RackUuid, pub sleds: BiHashMap, } +impl From for MultirackJoinServiceStatus { + fn from(value: StartSledAgentsStatus) -> Self { + let sled_id = value.this_nodes_sled_id; + let rack_id = value.rack_id; + let rack_subnet = Ipv6Subnet::::new( + value + .sleds + .iter() + .next() + .expect("at least one sled") + .sled_subnet + .net() + .addr(), + ); + let sled_agents = value + .sleds + .into_iter() + .map(|s| sled_agent_types::multirack::SledAgentInfo { + baseboard_id: s.baseboard_id, + sled_id: s.sled_id, + address: *get_sled_address(s.sled_subnet).ip(), + start_state: s.start_state, + }) + .collect(); + + MultirackJoinServiceStatus { + sled_id, + rack_id, + rack_subnet, + sled_agents, + } + } +} + impl StartSledAgentsStatus { // Create the allocations necessary to start a sled-agent and return a // preliminary state. // // This should only be called once. - pub fn assign_ids_and_subnets(req: MultirackJoinRequest) -> Self { + pub fn assign_ids_and_subnets( + this_nodes_baseboard_id: &BaseboardId, + rack_id: RackUuid, + req: MultirackJoinRequest, + ) -> Self { let rack_subnet = Ipv6Subnet::::new( req.rack_network_config.rack_subnet.addr(), ); + + // This is guaranteed to be filled in, because validating a trust + // quorum reconfiguration request ensures that this node is a member of + // `trust_quorum_peers`. + let mut this_nodes_sled_id = None; + let sleds = req .trust_quorum_peers .into_iter() .enumerate() - .map(|(idx, baseboard_id)| SledAgentInfo { - baseboard_id, - sled_id: SledUuid::new_v4(), - sled_subnet: get_64_subnet( - rack_subnet, - u8::try_from(idx + 1).expect("too many sleds"), - ), - start_state: SledAgentStartState::InProgress { - last_error: None, - }, + .map(|(idx, baseboard_id)| { + let sled_id = SledUuid::new_v4(); + if baseboard_id == *this_nodes_baseboard_id { + this_nodes_sled_id = Some(sled_id); + } + + SledAgentInfo { + baseboard_id, + sled_id, + sled_subnet: get_64_subnet( + rack_subnet, + u8::try_from(idx + 1).expect("too many sleds"), + ), + start_state: SledAgentStartState::InProgress { + last_error: None, + }, + } }) .collect(); - StartSledAgentsStatus { sleds } + // SAFETY: Construction of the trust quorum ensures this node is a member + let this_nodes_sled_id = this_nodes_sled_id.unwrap(); + + StartSledAgentsStatus { this_nodes_sled_id, rack_id, sleds } } } @@ -525,9 +576,9 @@ pub enum MultirackJoinServiceState { TrustQuorumReconfigure(TqReconfigureMsg), TrustQuorumPreparing(CoordinatorStatus), TrustQuorumCommitting(CommitState), - ConfigureNetworking, + ConfigureNetworking(Option), StartSledAgents(StartSledAgentsStatus), - WaitForCompletion, + WaitForCompletion(StartSledAgentsStatus), Completed, Failed { message: String }, InvalidMembershipSize { message: String }, diff --git a/sled-agent/multirack-join/src/lib.rs b/sled-agent/multirack-join/src/lib.rs index 5bcd4d31114..9bc10209c33 100644 --- a/sled-agent/multirack-join/src/lib.rs +++ b/sled-agent/multirack-join/src/lib.rs @@ -17,20 +17,16 @@ extern crate slog; use bootstrap_agent_lockstep_types::{ CommitState, MultirackJoinRequest, MultirackJoinServiceState, - SledAgentInfo, SledAgentStartState, StartSledAgentsStatus, + SledAgentInfo, StartSledAgentsStatus, }; -use iddqd::BiHashMap; use nexus_types::trust_quorum::TrustQuorumConfig; -use omicron_common::address::{ - BOOTSTRAP_AGENT_RACK_INIT_PORT, Ipv6Subnet, RACK_PREFIX_LENGTH, - get_sled_address, -}; +use omicron_common::address::BOOTSTRAP_AGENT_RACK_INIT_PORT; use omicron_ledger::{self as ledger}; use omicron_uuid_kinds::RackUuid; use sled_agent_bootstrap_common::sprockets::SprocketsClient; use sled_agent_bootstrap_common::{RssContext, RunRssError}; use sled_agent_types::early_networking::EarlyNetworkConfigEnvelope; -use sled_agent_types::multirack; +use sled_agent_types::multirack::SledAgentStartState; use sled_agent_types::rack_init::rack_init_bootstore_generation; use sled_agent_types::sled::{ StartSledAgentRequest, StartSledAgentRequestBody, @@ -165,8 +161,6 @@ pub struct MultirackJoinServiceHandle { tokio::task::JoinHandle>, pub input_tx: watch::Sender, pub output_rx: watch::Receiver, - pub status_rx: - watch::Receiver>, // Once we've initialized trust quorum, we no longer allow membership // changes. This allows helpful responses to user requests in case a @@ -187,7 +181,6 @@ impl MultirackJoinServiceHandle { let (input_tx, input_rx) = watch::channel(request); let state = MultirackJoinServiceState::Requested; let (output_tx, output_rx) = watch::channel(state.clone()); - let (status_tx, status_rx) = watch::channel(None); let membership_change_still_possible = Arc::new(AtomicBool::new(true)); let mcsp = membership_change_still_possible.clone(); let (rack_join_successful_tx, rack_join_successful_rx) = @@ -202,7 +195,6 @@ impl MultirackJoinServiceHandle { ctx, input_rx, output_tx, - status_tx, membership_change_still_possible: mcsp, rack_join_successful_rx, bootstore_generation, @@ -214,7 +206,6 @@ impl MultirackJoinServiceHandle { join_handle, input_tx, output_rx, - status_rx, membership_change_still_possible, rack_join_successful_tx, } @@ -227,7 +218,6 @@ struct MultirackJoinServiceTask { ctx: RssContext, input_rx: watch::Receiver, output_tx: watch::Sender, - status_tx: watch::Sender>, // Once we've initialized trust quorum, we no longer allow membership // changes. This allows helpful responses to user requests in case a @@ -299,8 +289,33 @@ impl MultirackJoinServiceTask { &mut self, ) -> Result<(), MultirackJoinServiceError> { loop { - self.output_tx.send_modify(|state| { - *state = MultirackJoinServiceState::WaitForCompletion + self.output_tx.send_modify(|state| match state { + MultirackJoinServiceState::ConfigureNetworking(Some( + status, + )) => { + *state = MultirackJoinServiceState::WaitForCompletion( + status.clone(), + ) + } + MultirackJoinServiceState::StartSledAgents(status) => { + *state = MultirackJoinServiceState::WaitForCompletion( + status.clone(), + ) + } + MultirackJoinServiceState::ConfigureNetworking(None) + | MultirackJoinServiceState::Uninitialized + | MultirackJoinServiceState::Requested + | MultirackJoinServiceState::Starting + | MultirackJoinServiceState::TrustQuorumReconfigure(_) + | MultirackJoinServiceState::TrustQuorumPreparing(_) + | MultirackJoinServiceState::TrustQuorumCommitting(_) + | MultirackJoinServiceState::WaitForCompletion(_) + | MultirackJoinServiceState::Completed + | MultirackJoinServiceState::Failed { .. } + | MultirackJoinServiceState::InvalidMembershipSize { .. } + | MultirackJoinServiceState::TaskPanicked => { + panic!("MultirackJoinService in wrong state: {:#?}", state); + } }); tokio::select! { @@ -333,8 +348,28 @@ impl MultirackJoinServiceTask { let rack_network_config = self.input_rx.borrow_and_update().rack_network_config.clone(); - self.output_tx.send_modify(|state| { - *state = MultirackJoinServiceState::ConfigureNetworking + self.output_tx.send_modify(|state| match state { + MultirackJoinServiceState::TrustQuorumCommitting(_) => { + *state = MultirackJoinServiceState::ConfigureNetworking(None) + } + MultirackJoinServiceState::WaitForCompletion(status) => { + *state = MultirackJoinServiceState::ConfigureNetworking(Some( + status.clone(), + )) + } + MultirackJoinServiceState::Uninitialized + | MultirackJoinServiceState::Requested + | MultirackJoinServiceState::Starting + | MultirackJoinServiceState::TrustQuorumReconfigure(_) + | MultirackJoinServiceState::TrustQuorumPreparing(_) + | MultirackJoinServiceState::ConfigureNetworking(_) + | MultirackJoinServiceState::StartSledAgents(_) + | MultirackJoinServiceState::Completed + | MultirackJoinServiceState::Failed { .. } + | MultirackJoinServiceState::InvalidMembershipSize { .. } + | MultirackJoinServiceState::TaskPanicked => { + panic!("MultirackJoinService in wrong state: {:#?}", state); + } }); let config = SystemNetworkingConfig { @@ -378,40 +413,15 @@ impl MultirackJoinServiceTask { // `changed()` wouldn't fire. let req = self.input_rx.borrow().clone(); let tq_members = req.trust_quorum_peers.clone(); - let rack_subnet = Ipv6Subnet::::new( - req.rack_network_config.rack_subnet.addr(), + let status = StartSledAgentsStatus::assign_ids_and_subnets( + self.ctx.trust_quorum_handle.baseboard_id(), + rack_id, + req, ); - let status = StartSledAgentsStatus::assign_ids_and_subnets(req); self.output_tx.send_modify(|state| { *state = MultirackJoinServiceState::StartSledAgents(status.clone()) }); - let sled_agents: BiHashMap = status - .sleds - .iter() - .map(|info| multirack::SledAgentInfo { - baseboard_id: info.baseboard_id.clone(), - sled_id: info.sled_id, - address: *get_sled_address(info.sled_subnet).ip(), - start_state: multirack::SledAgentStartState::InProgress { - last_error: None, - }, - }) - .collect(); - - if let Some(info) = - status.sleds.get1(self.ctx.trust_quorum_handle.baseboard_id()) - { - self.status_tx.send_replace(Some( - multirack::MultirackJoinServiceStatus { - sled_id: info.sled_id, - rack_id, - rack_subnet, - sled_agents, - }, - )); - } - let mut bootstrap_ips: BTreeMap<_, _> = self .ctx .trust_quorum_handle @@ -473,14 +483,6 @@ impl MultirackJoinServiceTask { let mut info = status.sleds.get1_mut(&baseboard_id).unwrap(); info.start_state = SledAgentStartState::Started; }); - self.status_tx.send_modify(|status| { - if let Some(status) = status - && let Some(mut info) = - status.sled_agents.get1_mut(&baseboard_id) - { - info.start_state = multirack::SledAgentStartState::Started; - } - }); } Ok(()) @@ -520,7 +522,6 @@ impl MultirackJoinServiceTask { ); let output_tx = self.output_tx.clone(); - let status_tx = self.status_tx.clone(); let mut retry_count = 0; async move { // Retry indefinitely to start sled-agent. @@ -548,20 +549,6 @@ impl MultirackJoinServiceTask { info.start_state = SledAgentStartState::InProgress { last_error: Some(errstr) }; }); - status_tx.send_modify(|status| { - let errstr = - InlineErrorChain::new(&err).to_string(); - if let Some(status) = status - && let Some(mut sled) = status - .sled_agents - .get1_mut(&info.baseboard_id) - { - sled.start_state = - multirack::SledAgentStartState::InProgress { - last_error: Some(errstr), - }; - } - }); // Only log every minute, given the 6 second sleep below if retry_count % 10 == 0 { warn!(log, "Failed to start sled agent"; &err); @@ -1138,7 +1125,12 @@ mod tests { rack_network_config: rack_network_config(), }; - let status = StartSledAgentsStatus::assign_ids_and_subnets(req.clone()); + let coordinator = req.trust_quorum_peers.first().unwrap(); + let status = StartSledAgentsStatus::assign_ids_and_subnets( + coordinator, + RackUuid::new_v4(), + req.clone(), + ); assert_eq!(status.sleds.len(), req.trust_quorum_peers.len()); let actual_sled_subnets: BTreeSet<_> = diff --git a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs index a4701bf84bc..d6c0413a9e9 100644 --- a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs +++ b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs @@ -121,7 +121,31 @@ impl BootstrapAgentLockstepApi for BootstrapAgentLockstepImpl { let ctx = rqctx.context(); let request = body.into_inner(); let params = RackInitializeRequestParams::new(request, SKIP_TIMESYNC); - let id = ctx.start_rack_initialize(params).map_err(HttpError::from)?; + let id = + ctx.start_rack_initialize(params).map_err(|err| match err { + RssAccessError::InitializationFailed { .. } + | RssAccessError::InitializationPanicked => { + HttpError::for_internal_error(err.to_string()) + } + RssAccessError::StillInitializing + | RssAccessError::AlreadyInitialized + | RssAccessError::MultirackJoinTaskExitedUnexpectedly + | RssAccessError::MultirackJoinPanicked + | RssAccessError::MultirackJoinInProgress + | RssAccessError::MultirackJoinFailed { .. } + | RssAccessError::MultirackJoinCompleted + | RssAccessError::MembershipChangeNoLongerAllowed + | RssAccessError::RackSubnetChangeNotAllowed + | RssAccessError::MultirackJoinNotYetStarted + | RssAccessError::MultirackJoinStatusNotYetAvailable => { + HttpError::for_client_error( + Some(err.error_code().to_string()), + dropshot::ClientErrorStatusCode::CONFLICT, + err.to_string(), + ) + } + })?; + Ok(HttpResponseOk(id)) } @@ -184,7 +208,30 @@ impl BootstrapAgentLockstepApi for BootstrapAgentLockstepImpl { ) -> Result, HttpError> { let ctx = rqctx.context(); let request = body.into_inner(); - let id = ctx.start_multirack_join(request)?; + let id = + ctx.start_multirack_join(request).map_err(|err| match err { + RssAccessError::MultirackJoinTaskExitedUnexpectedly + | RssAccessError::MultirackJoinPanicked => { + HttpError::for_internal_error(err.to_string()) + } + RssAccessError::StillInitializing + | RssAccessError::InitializationFailed { .. } + | RssAccessError::InitializationPanicked + | RssAccessError::AlreadyInitialized + | RssAccessError::MultirackJoinInProgress + | RssAccessError::MultirackJoinFailed { .. } + | RssAccessError::MultirackJoinCompleted + | RssAccessError::MembershipChangeNoLongerAllowed + | RssAccessError::RackSubnetChangeNotAllowed + | RssAccessError::MultirackJoinNotYetStarted + | RssAccessError::MultirackJoinStatusNotYetAvailable => { + HttpError::for_client_error( + Some(err.error_code().to_string()), + dropshot::ClientErrorStatusCode::CONFLICT, + err.to_string(), + ) + } + })?; Ok(HttpResponseOk(id)) } diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index 8dd7e407698..13983f7f069 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -4,12 +4,12 @@ //! Internal API for rack-level bootstrap agent operations. +use crate::bootstrap::RssAccessError::MultirackJoinNotYetStarted; use crate::bootstrap::rss_handle::run_rss; use bootstrap_agent_lockstep_types::MultirackJoinRequest; use bootstrap_agent_lockstep_types::MultirackJoinServiceState; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::RssStep; -use dropshot::HttpError; use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; use sled_agent_bootstrap_common::RssContext; @@ -94,49 +94,6 @@ impl RssAccessError { } } -impl From for HttpError { - fn from(err: RssAccessError) -> Self { - // Most variants of RssAccessError report states that conflict with the - // requested rack operation in some fashion: - // - // * `StillInitializing`/`MultirackJoinInProgress` mean that an - // operation is in progress. - // * `AlreadyInitialized`/`MultirackJoinCompleted` mean that we're - // already at the requested end state. (Note that we don't try and - // be idempotent here -- to do so we'd have to ensure the actual RSS - // config is the same, which is tricky. Instead, clients should look - // at the error code to determine what to do.) - // * `MultirackJoinNotYetStarted` means it can't complete when signalled. - // * The other states are terminal states that need operator intervention. - // - // We map all of these states to 409 Conflict errors, and (since this is - // an internal API) expose the error text to the client. - match err { - RssAccessError::MultirackJoinTaskExitedUnexpectedly => { - HttpError::for_internal_error(err.to_string()) - } - RssAccessError::StillInitializing - | RssAccessError::InitializationFailed { .. } - | RssAccessError::InitializationPanicked - | RssAccessError::AlreadyInitialized - | RssAccessError::MultirackJoinInProgress - | RssAccessError::MultirackJoinFailed { .. } - | RssAccessError::MultirackJoinPanicked - | RssAccessError::MultirackJoinCompleted - | RssAccessError::MembershipChangeNoLongerAllowed - | RssAccessError::RackSubnetChangeNotAllowed - | RssAccessError::MultirackJoinNotYetStarted - | RssAccessError::MultirackJoinStatusNotYetAvailable => { - HttpError::for_client_error( - Some(err.error_code().to_string()), - dropshot::ClientErrorStatusCode::CONFLICT, - err.to_string(), - ) - } - } - } -} - /// A mechanism for sled-agent to access its local multirack join service task. #[derive(Clone)] pub(crate) struct MultirackJoinAccessor(RssAccess); @@ -157,7 +114,7 @@ impl MultirackJoinAccessor { /// running. pub fn multirack_join_service_status( &self, - ) -> Result, RssAccessError> { + ) -> Result { self.0.multirack_join_service_status() } } @@ -407,7 +364,6 @@ impl RssAccess { join_handle, input_tx, output_rx, - status_rx, membership_change_still_possible, rack_join_successful_tx, } = MultirackJoinServiceHandle::spawn(ctx, request); @@ -415,7 +371,6 @@ impl RssAccess { id, input_tx, output_rx, - status_rx, membership_change_still_possible, rack_join_successful_tx: Some(rack_join_successful_tx), }; @@ -530,24 +485,70 @@ impl RssAccess { pub(crate) fn multirack_join_service_status( &self, - ) -> Result, RssAccessError> { + ) -> Result { match &*self.status.lock().unwrap() { - RssStatus::MultirackJoinInProgress { status_rx, .. } => { - match status_rx.borrow().clone() { - Some(status) => Ok(Some(status)), - None => { + RssStatus::MultirackJoinInProgress { output_rx, .. } => { + match output_rx.borrow().clone() { + MultirackJoinServiceState::StartSledAgents(status) => { + Ok(status.into()) + } + MultirackJoinServiceState::ConfigureNetworking(Some( + status, + )) => Ok(status.into()), + MultirackJoinServiceState::WaitForCompletion(status) => { + Ok(status.into()) + } + MultirackJoinServiceState::ConfigureNetworking(None) + | MultirackJoinServiceState::Requested + | MultirackJoinServiceState::Starting + | MultirackJoinServiceState::TrustQuorumReconfigure(_) + | MultirackJoinServiceState::TrustQuorumPreparing(_) + | MultirackJoinServiceState::TrustQuorumCommitting(_) => { Err(RssAccessError::MultirackJoinStatusNotYetAvailable) } + MultirackJoinServiceState::Uninitialized => { + Err(RssAccessError::MultirackJoinNotYetStarted) + } + MultirackJoinServiceState::Completed => { + Err(RssAccessError::MultirackJoinCompleted) + } + MultirackJoinServiceState::Failed { message } => { + Err(RssAccessError::MultirackJoinFailed { message }) + } + MultirackJoinServiceState::InvalidMembershipSize { + message, + } => Err(RssAccessError::MultirackJoinFailed { message }), + MultirackJoinServiceState::TaskPanicked => { + Err(RssAccessError::MultirackJoinPanicked) + } } } - RssStatus::Uninitialized - | RssStatus::Initialized { .. } - | RssStatus::Initializing { .. } - | RssStatus::InitializationFailed { .. } - | RssStatus::InitializationPanicked { .. } - | RssStatus::MultirackJoinCompleted { .. } - | RssStatus::MultirackJoinFailed { .. } - | RssStatus::MultirackJoinPanicked { .. } => Ok(None), + RssStatus::Uninitialized => Err(MultirackJoinNotYetStarted), + RssStatus::Initialized { .. } => { + Err(RssAccessError::InitializationPanicked) + } + RssStatus::Initializing { .. } => { + Err(RssAccessError::StillInitializing) + } + RssStatus::InitializationFailed { err, .. } => { + Err(RssAccessError::InitializationFailed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::InitializationPanicked { .. } => { + Err(RssAccessError::InitializationPanicked) + } + RssStatus::MultirackJoinCompleted { .. } => { + Err(RssAccessError::MultirackJoinCompleted) + } + RssStatus::MultirackJoinFailed { err, .. } => { + Err(RssAccessError::MultirackJoinFailed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::MultirackJoinPanicked { .. } => { + Err(RssAccessError::InitializationPanicked) + } } } @@ -623,7 +624,6 @@ enum RssStatus { id: MultirackJoinUuid, input_tx: watch::Sender, output_rx: watch::Receiver, - status_rx: watch::Receiver>, membership_change_still_possible: Arc, rack_join_successful_tx: Option>, }, @@ -643,45 +643,3 @@ enum RssStatus { id: MultirackJoinUuid, }, } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rss_access_error_to_http_error() { - let cases = [ - RssAccessError::StillInitializing, - RssAccessError::InitializationFailed { - message: "boom".to_string(), - }, - RssAccessError::InitializationPanicked, - RssAccessError::AlreadyInitialized, - ]; - for err in cases { - let error_code = err.error_code(); - let display = err.to_string(); - let http_error = HttpError::from(err); - assert_eq!( - http_error.status_code.as_u16(), - 409, - "{error_code}: expected 409 Conflict" - ); - assert_eq!( - http_error.error_code.as_deref(), - Some(error_code), - "{error_code}: error_code should be the variant name" - ); - assert_eq!( - http_error.external_message, display, - "{error_code}: full error text should be exposed to the \ - client" - ); - assert_eq!( - http_error.internal_message, display, - "{error_code}: internal message should match the error \ - text" - ); - } - } -} diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 43476ac27f8..90cc020f895 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -5,6 +5,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API use super::sled_agent::SledAgent; +use crate::bootstrap::RssAccessError; use crate::sled_agent::Error as SledAgentError; use crate::support_bundle::storage::SupportBundleQueryType; use crate::zone_bundle::BundleError; @@ -1861,13 +1862,54 @@ impl SledAgentApi for SledAgentImpl { let sa = request_context.context(); sa.latencies() .instrument_dropshot_handler(&request_context, async { - match sa.multirack_join_service_status()? { - Some(status) => Ok(HttpResponseOk(status)), - None => Err(HttpError::for_not_found( - None, - "No multirack join service running".to_string(), - )), - } + let res = sa.multirack_join_service_status(); + let status = res.map_err(|err| match err { + // Errors returned from multirack join should be a 500. + RssAccessError::MultirackJoinTaskExitedUnexpectedly + | RssAccessError::MultirackJoinFailed { .. } + | RssAccessError::MultirackJoinPanicked => { + HttpError::for_internal_error(err.to_string()) + } + + // RSS running/having run or Multirack join not started + // should return a 404. + // + // We also return a 404 if the multirack join service has + // completed. In this case Nexus will have already recorded + // the results to CRDB, and so there is no real error here. + // We no longer have the status to report anyway, and it + // seems overly complicated to keep track of it and map it + // to an option which the consumer would have to match on. + // This is especially true since we expect the consumer to + // be a background task that next time through will see the + // value in the DB and not make another request. + RssAccessError::MultirackJoinNotYetStarted + | RssAccessError::StillInitializing + | RssAccessError::InitializationFailed { .. } + | RssAccessError::InitializationPanicked + | RssAccessError::AlreadyInitialized + | RssAccessError::MultirackJoinCompleted => { + HttpError::for_not_found( + None, + "No multirack join service running".to_string(), + ) + } + + // If we don't yet have the status available, but we will + // soon we should return a 409. + RssAccessError::MultirackJoinInProgress + | RssAccessError::MembershipChangeNoLongerAllowed + | RssAccessError::RackSubnetChangeNotAllowed + | RssAccessError::MultirackJoinStatusNotYetAvailable => { + HttpError::for_client_error( + Some(err.error_code().to_string()), + dropshot::ClientErrorStatusCode::CONFLICT, + err.to_string(), + ) + } + })?; + + Ok(HttpResponseOk(status)) }) .await } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 858afdb6625..b5c869ea78f 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -5,7 +5,7 @@ //! Sled agent implementation use crate::artifact_store::{ArtifactStore, SledAgentArtifactStoreWrapper}; -use crate::bootstrap::MultirackJoinAccessor; +use crate::bootstrap::{MultirackJoinAccessor, RssAccessError}; use crate::config::Config; use crate::hardware_monitor::HardwareMonitorHandle; use crate::instance_manager::InstanceManager; @@ -404,7 +404,6 @@ impl From for dropshot::HttpError { let err = omicron_common::api::external::Error::from(err); err.into() } - Error::MultirackJoin(err) => err.into(), err @ (Error::BootDiskNotFound | Error::Config(_) | Error::BackingFs(_) @@ -426,6 +425,7 @@ impl From for dropshot::HttpError { | Error::UnexpectedRevision(_) | Error::RepoDepotStart(_) | Error::TimeNotSynchronized + | Error::MultirackJoin(_) | Error::Rot(_)) => HttpError::for_internal_error( InlineErrorChain::new(&err).to_string(), ), @@ -1729,7 +1729,7 @@ impl SledAgent { pub(crate) fn multirack_join_service_status( &self, - ) -> Result, Error> { + ) -> Result { Ok(self .inner .multirack_join_completer diff --git a/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs b/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs index 62bd591fcda..3bb8ec12cd2 100644 --- a/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs +++ b/sled-agent/types/versions/src/add_multirack_join_service_status/multirack.rs @@ -45,6 +45,7 @@ impl BiHashItem for SledAgentInfo { bi_upcast!(); } +// Information about the state of a sled agent start request #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "snake_case", tag = "type")] pub enum SledAgentStartState { From 5991029fb9190c4c0e1e69596a5bd7c4cdc3cac4 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 21 Oct 2026 20:49:25 +0000 Subject: [PATCH 3/5] renamearooni --- sled-agent/src/bootstrap/server.rs | 14 +++++++------- sled-agent/src/server.rs | 4 ++-- sled-agent/src/sled_agent.rs | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 4f5cc481ea1..a6fb65745cd 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -204,7 +204,7 @@ impl Server { let rss_access = RssAccess::new(maybe_ledger.is_some()); // Provide a mechanism for multirack join requests to be completed by sled-agent. - let multirack_join_completer = + let multirack_join_accessor = MultirackJoinAccessor::new(rss_access.clone()); // Start the bootstrap dropshot server. @@ -259,7 +259,7 @@ impl Server { long_running_task_handles.clone(), config_reconciler_spawn_token, service_manager.clone(), - multirack_join_completer.clone(), + multirack_join_accessor.clone(), &base_log, &startup_log, ) @@ -291,7 +291,7 @@ impl Server { service_manager, _sprockets_server_handle: sprockets_server_handle, base_log, - multirack_join_completer, + multirack_join_accessor, }; let inner_task = tokio::spawn(inner.run()); @@ -370,7 +370,7 @@ async fn start_sled_agent( long_running_task_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, service_manager: ServiceManager, - multirack_join_completer: MultirackJoinAccessor, + multirack_join_accessor: MultirackJoinAccessor, base_log: &Logger, log: &Logger, ) -> Result { @@ -428,7 +428,7 @@ async fn start_sled_agent( long_running_task_handles.clone(), config_reconciler_spawn_token, service_manager, - multirack_join_completer, + multirack_join_accessor, ) .await .map_err(SledAgentServerStartError::FailedStartingServer)?; @@ -520,7 +520,7 @@ struct Inner { service_manager: ServiceManager, _sprockets_server_handle: JoinHandle<()>, base_log: Logger, - multirack_join_completer: MultirackJoinAccessor, + multirack_join_accessor: MultirackJoinAccessor, } impl Inner { @@ -560,7 +560,7 @@ impl Inner { self.long_running_task_handles.clone(), config_reconciler_spawn_token, self.service_manager.clone(), - self.multirack_join_completer.clone(), + self.multirack_join_accessor.clone(), &self.base_log, &log, ) diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index dd499b496f6..d46c6d91c04 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -44,7 +44,7 @@ impl Server { long_running_tasks_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, services: ServiceManager, - multirack_join_completer: MultirackJoinAccessor, + multirack_join_accessor: MultirackJoinAccessor, ) -> Result { info!(log, "setting up sled agent server"); @@ -67,7 +67,7 @@ impl Server { services, long_running_tasks_handles, config_reconciler_spawn_token, - multirack_join_completer, + multirack_join_accessor, ) .await .map_err(|e| InlineErrorChain::new(&e).to_string())?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b5c869ea78f..1a33460e142 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -528,8 +528,8 @@ struct SledAgentInner { // Handle to access reference measurements for Sprockets/TrustQuorum measurements: Arc, - // A mechanism to complete a multirack join - multirack_join_completer: MultirackJoinAccessor, + // A mechanism to access a multirack join service task + multirack_join_accessor: MultirackJoinAccessor, } impl SledAgentInner { @@ -562,7 +562,7 @@ impl SledAgent { services: ServiceManager, long_running_task_handles: LongRunningTaskHandles, config_reconciler_spawn_token: ConfigReconcilerSpawnToken, - multirack_join_completer: MultirackJoinAccessor, + multirack_join_accessor: MultirackJoinAccessor, ) -> Result { // Pass the "parent_log" to all subcomponents that want to set their own // "component" value. @@ -892,7 +892,7 @@ impl SledAgent { metrics_manager, repo_depot, measurements: long_running_task_handles.measurements.clone(), - multirack_join_completer, + multirack_join_accessor, }), log: log.clone(), sprockets: config.sprockets.clone(), @@ -1723,7 +1723,7 @@ impl SledAgent { /// Complete a multirack join operation pub(crate) fn complete_multirack_join(&self) -> Result<(), Error> { - self.inner.multirack_join_completer.complete_multirack_join()?; + self.inner.multirack_join_accessor.complete_multirack_join()?; Ok(()) } @@ -1732,7 +1732,7 @@ impl SledAgent { ) -> Result { Ok(self .inner - .multirack_join_completer + .multirack_join_accessor .multirack_join_service_status()?) } } From 95aa87b5990b16fd2dbf3eb3496d6b8c2e77d97a Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 21 Oct 2026 20:54:26 +0000 Subject: [PATCH 4/5] clippy --- sled-agent/src/sled_agent.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 1a33460e142..14d430dab3f 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -1730,10 +1730,7 @@ impl SledAgent { pub(crate) fn multirack_join_service_status( &self, ) -> Result { - Ok(self - .inner - .multirack_join_accessor - .multirack_join_service_status()?) + self.inner.multirack_join_accessor.multirack_join_service_status() } } From 59f520a5b04300fff06d24936da8626077e3b964 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 21 Oct 2026 21:12:03 +0000 Subject: [PATCH 5/5] fix progenitor error --- openapi/bootstrap-agent-lockstep.json | 99 +++++++++++++++---- .../bootstrap-agent-lockstep-types/src/lib.rs | 12 ++- sled-agent/multirack-join/src/lib.rs | 22 +++-- sled-agent/src/bootstrap/rack_ops.rs | 10 +- 4 files changed, 107 insertions(+), 36 deletions(-) diff --git a/openapi/bootstrap-agent-lockstep.json b/openapi/bootstrap-agent-lockstep.json index 06050ce661e..8581583457a 100644 --- a/openapi/bootstrap-agent-lockstep.json +++ b/openapi/bootstrap-agent-lockstep.json @@ -2343,6 +2343,14 @@ { "type": "object", "properties": { + "start_sled_agent_status": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/StartSledAgentsStatus" + } + ] + }, "state": { "type": "string", "enum": [ @@ -2357,6 +2365,9 @@ { "type": "object", "properties": { + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, "sleds": { "title": "BiHashMap", "x-rust-type": { @@ -2380,25 +2391,57 @@ "enum": [ "start_sled_agents" ] + }, + "this_nodes_sled_id": { + "$ref": "#/components/schemas/SledUuid" } }, "required": [ + "rack_id", "sleds", - "state" + "state", + "this_nodes_sled_id" ] }, { "type": "object", "properties": { + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "sleds": { + "title": "BiHashMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/SledAgentInfo" + } + ], + "path": "iddqd::BiHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/SledAgentInfo" + }, + "uniqueItems": true + }, "state": { "type": "string", "enum": [ "wait_for_completion" ] + }, + "this_nodes_sled_id": { + "$ref": "#/components/schemas/SledUuid" } }, "required": [ - "state" + "rack_id", + "sleds", + "state", + "this_nodes_sled_id" ] }, { @@ -4018,24 +4061,6 @@ "required": [ "type" ] - }, - { - "type": "object", - "properties": { - "reason": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "failed" - ] - } - }, - "required": [ - "reason", - "type" - ] } ] }, @@ -4048,6 +4073,40 @@ "type": "string", "format": "uuid" }, + "StartSledAgentsStatus": { + "type": "object", + "properties": { + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "sleds": { + "title": "BiHashMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/SledAgentInfo" + } + ], + "path": "iddqd::BiHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/SledAgentInfo" + }, + "uniqueItems": true + }, + "this_nodes_sled_id": { + "$ref": "#/components/schemas/SledUuid" + } + }, + "required": [ + "rack_id", + "sleds", + "this_nodes_sled_id" + ] + }, "SwitchSlot": { "description": "Identifies switch physical location", "oneOf": [ diff --git a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs index df7f0d35840..690afc5b342 100644 --- a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs @@ -576,11 +576,17 @@ pub enum MultirackJoinServiceState { TrustQuorumReconfigure(TqReconfigureMsg), TrustQuorumPreparing(CoordinatorStatus), TrustQuorumCommitting(CommitState), - ConfigureNetworking(Option), + ConfigureNetworking { + start_sled_agent_status: Option, + }, StartSledAgents(StartSledAgentsStatus), WaitForCompletion(StartSledAgentsStatus), Completed, - Failed { message: String }, - InvalidMembershipSize { message: String }, + Failed { + message: String, + }, + InvalidMembershipSize { + message: String, + }, TaskPanicked, } diff --git a/sled-agent/multirack-join/src/lib.rs b/sled-agent/multirack-join/src/lib.rs index 9bc10209c33..577255039fd 100644 --- a/sled-agent/multirack-join/src/lib.rs +++ b/sled-agent/multirack-join/src/lib.rs @@ -290,9 +290,9 @@ impl MultirackJoinServiceTask { ) -> Result<(), MultirackJoinServiceError> { loop { self.output_tx.send_modify(|state| match state { - MultirackJoinServiceState::ConfigureNetworking(Some( - status, - )) => { + MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: Some(status), + } => { *state = MultirackJoinServiceState::WaitForCompletion( status.clone(), ) @@ -302,7 +302,9 @@ impl MultirackJoinServiceTask { status.clone(), ) } - MultirackJoinServiceState::ConfigureNetworking(None) + MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: None, + } | MultirackJoinServiceState::Uninitialized | MultirackJoinServiceState::Requested | MultirackJoinServiceState::Starting @@ -350,19 +352,21 @@ impl MultirackJoinServiceTask { self.output_tx.send_modify(|state| match state { MultirackJoinServiceState::TrustQuorumCommitting(_) => { - *state = MultirackJoinServiceState::ConfigureNetworking(None) + *state = MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: None, + } } MultirackJoinServiceState::WaitForCompletion(status) => { - *state = MultirackJoinServiceState::ConfigureNetworking(Some( - status.clone(), - )) + *state = MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: Some(status.clone()), + } } MultirackJoinServiceState::Uninitialized | MultirackJoinServiceState::Requested | MultirackJoinServiceState::Starting | MultirackJoinServiceState::TrustQuorumReconfigure(_) | MultirackJoinServiceState::TrustQuorumPreparing(_) - | MultirackJoinServiceState::ConfigureNetworking(_) + | MultirackJoinServiceState::ConfigureNetworking { .. } | MultirackJoinServiceState::StartSledAgents(_) | MultirackJoinServiceState::Completed | MultirackJoinServiceState::Failed { .. } diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index 13983f7f069..3059a0b0374 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -492,13 +492,15 @@ impl RssAccess { MultirackJoinServiceState::StartSledAgents(status) => { Ok(status.into()) } - MultirackJoinServiceState::ConfigureNetworking(Some( - status, - )) => Ok(status.into()), + MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: Some(status), + } => Ok(status.into()), MultirackJoinServiceState::WaitForCompletion(status) => { Ok(status.into()) } - MultirackJoinServiceState::ConfigureNetworking(None) + MultirackJoinServiceState::ConfigureNetworking { + start_sled_agent_status: None, + } | MultirackJoinServiceState::Requested | MultirackJoinServiceState::Starting | MultirackJoinServiceState::TrustQuorumReconfigure(_)