From 129f5ae3ea0f5607a6c8a7ee99d6370bd92eede2 Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 29 Oct 2026 15:11:53 -0700 Subject: [PATCH 1/2] [sled-agent] use wrapper structs for trust quorum responses --- Cargo.lock | 1 + .../src/test_util/host_phase_2_test_state.rs | 16 ++-- .../src/app/background/tasks/trust_quorum.rs | 10 ++- .../sled-agent-60.0.0-f9d5fe.json.gitstub | 1 + ...5fe.json => sled-agent-61.0.0-9c9968.json} | 36 +++++++- openapi/sled-agent/sled-agent-latest.json | 2 +- sled-agent/api/Cargo.toml | 1 + sled-agent/api/src/lib.rs | 90 +++++++++++++++++-- sled-agent/src/http_entrypoints.rs | 19 ++-- sled-agent/src/sim/http_entrypoints.rs | 10 ++- sled-agent/types/versions/src/latest.rs | 3 + sled-agent/types/versions/src/lib.rs | 2 + .../trust_quorum_nullable_responses/mod.rs | 11 +++ .../trust_quorum.rs | 25 ++++++ 14 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json.gitstub rename openapi/sled-agent/{sled-agent-60.0.0-f9d5fe.json => sled-agent-61.0.0-9c9968.json} (99%) create mode 100644 sled-agent/types/versions/src/trust_quorum_nullable_responses/mod.rs create mode 100644 sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs diff --git a/Cargo.lock b/Cargo.lock index 0dbe99f7449..7bfe5131e6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14380,6 +14380,7 @@ dependencies = [ "slog-error-chain 0.1.0 (git+https://github.com/oxidecomputer/slog-error-chain?branch=main)", "static_assertions", "trust-quorum-types", + "trust-quorum-types-versions", "uuid", "x509-cert", ] diff --git a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs index de81f420b3b..ce3292fbfde 100644 --- a/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs +++ b/nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs @@ -264,6 +264,8 @@ mod api_impl { use sled_agent_types::support_bundle::SupportBundleMetadata; use sled_agent_types::support_bundle::SupportBundlePathParam; use sled_agent_types::support_bundle::SupportBundleTransferQueryParams; + use sled_agent_types::trust_quorum::TrustQuorumCoordinatorStatusResponse; + use sled_agent_types::trust_quorum::TrustQuorumNetworkConfigResponse; use sled_agent_types::zone_bundle::BundleUtilization; use sled_agent_types::zone_bundle::CleanupContext; use sled_agent_types::zone_bundle::CleanupContextUpdate; @@ -1003,9 +1005,7 @@ mod api_impl { async fn trust_quorum_coordinator_status( _request_context: RequestContext, ) -> Result< - HttpResponseOk< - Option, - >, + HttpResponseOk, HttpError, > { unimplemented!() @@ -1065,14 +1065,8 @@ mod api_impl { async fn trust_quorum_network_config_get( _request_context: RequestContext, - ) -> Result< - HttpResponseOk< - Option< - sled_agent_types::trust_quorum::TrustQuorumNetworkConfig, - >, - >, - HttpError, - > { + ) -> Result, HttpError> + { unimplemented!() } diff --git a/nexus/src/app/background/tasks/trust_quorum.rs b/nexus/src/app/background/tasks/trust_quorum.rs index 849cd13c6fe..c1ac4e5938d 100644 --- a/nexus/src/app/background/tasks/trust_quorum.rs +++ b/nexus/src/app/background/tasks/trust_quorum.rs @@ -343,7 +343,15 @@ async fn prepare( config.coordinator ) })? - .into_inner(); + .into_inner() + .status + .ok_or_else(|| { + anyhow!( + "Coordinator {} reported no coordinator status: it is not \ + currently coordinating a reconfiguration.", + config.coordinator + ) + })?; // Write state back to DB let state = datastore diff --git a/openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json.gitstub b/openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json.gitstub new file mode 100644 index 00000000000..5133aa0ddd9 --- /dev/null +++ b/openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json.gitstub @@ -0,0 +1 @@ +d31af1e7ce811874957823cb8f731b6f10e35a1b:openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json diff --git a/openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json b/openapi/sled-agent/sled-agent-61.0.0-9c9968.json similarity index 99% rename from openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json rename to openapi/sled-agent/sled-agent-61.0.0-9c9968.json index 55bc3013457..b8e26023d66 100644 --- a/openapi/sled-agent/sled-agent-60.0.0-f9d5fe.json +++ b/openapi/sled-agent/sled-agent-61.0.0-9c9968.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "60.0.0" + "version": "61.0.0" }, "paths": { "/artifacts": { @@ -1691,7 +1691,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CoordinatorStatus" + "$ref": "#/components/schemas/TrustQuorumCoordinatorStatusResponse" } } } @@ -1715,7 +1715,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TrustQuorumNetworkConfig" + "$ref": "#/components/schemas/TrustQuorumNetworkConfigResponse" } } } @@ -10422,6 +10422,21 @@ "rack_network_config" ] }, + "TrustQuorumCoordinatorStatusResponse": { + "description": "Response type for `trust_quorum_coordinator_status`.", + "type": "object", + "properties": { + "status": { + "nullable": true, + "description": "The coordinator status, or `None` if this node is not currently coordinating a reconfiguration.", + "allOf": [ + { + "$ref": "#/components/schemas/CoordinatorStatus" + } + ] + } + } + }, "TrustQuorumNetworkConfig": { "description": "Network configuration used to bring up the control plane.\n\nThis type mirrors `bootstore::schemes::v0::NetworkConfig` but adds `JsonSchema` for API compatibility.", "type": "object", @@ -10441,6 +10456,21 @@ "generation" ] }, + "TrustQuorumNetworkConfigResponse": { + "description": "Response type for `trust_quorum_network_config`.", + "type": "object", + "properties": { + "config": { + "nullable": true, + "description": "The trust quorum network config, or `None` if a config has not been loaded or received since Sled Agent started.", + "allOf": [ + { + "$ref": "#/components/schemas/TrustQuorumNetworkConfig" + } + ] + } + } + }, "TxEqConfig": { "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", "type": "object", diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index dc862f3b817..9e4ea7eb247 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-60.0.0-f9d5fe.json \ No newline at end of file +sled-agent-61.0.0-9c9968.json \ No newline at end of file diff --git a/sled-agent/api/Cargo.toml b/sled-agent/api/Cargo.toml index 8efa42ead18..683a7e51217 100644 --- a/sled-agent/api/Cargo.toml +++ b/sled-agent/api/Cargo.toml @@ -27,5 +27,6 @@ sled-hardware-types.workspace = true slog-error-chain.workspace = true static_assertions.workspace = true trust-quorum-types.workspace = true +trust-quorum-types-versions.workspace = true uuid.workspace = true x509-cert.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index cc020387a0f..88d829f8007 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -6,10 +6,10 @@ use std::collections::BTreeMap; use camino::Utf8PathBuf; use dropshot::{ - Body, FreeformBody, Header, HttpError, HttpResponseAccepted, - HttpResponseCreated, HttpResponseDeleted, HttpResponseHeaders, - HttpResponseOk, HttpResponseUpdatedNoContent, Path, Query, RequestContext, - StreamingBody, TypedBody, + Body, ClientErrorStatusCode, FreeformBody, Header, HttpError, + HttpResponseAccepted, HttpResponseCreated, HttpResponseDeleted, + HttpResponseHeaders, HttpResponseOk, HttpResponseUpdatedNoContent, Path, + Query, RequestContext, StreamingBody, TypedBody, }; use dropshot_api_manager_types::api_versions; use omicron_common::api::internal::shared::{ @@ -17,9 +17,9 @@ use omicron_common::api::internal::shared::{ SledIdentifiers, VirtualNetworkInterfaceHost, }; use sled_agent_types_versions::{ - latest, v1, v11, v14, v18, v20, v22, v24, v25, v26, v28, v29, v30, v31, - v32, v33, v34, v37, v39, v40, v41, v42, v43, v44, v46, v47, v48, v49, v50, - v51, v52, v53, v55, + latest, v1, v11, v14, v15, v18, v20, v22, v24, v25, v26, v28, v29, v30, + v31, v32, v33, v34, v37, v39, v40, v41, v42, v43, v44, v46, v47, v48, v49, + v50, v51, v52, v53, v55, }; use sled_diagnostics::SledDiagnosticsQueryOutput; use slog_error_chain::InlineErrorChain; @@ -36,6 +36,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (61, TRUST_QUORUM_NULLABLE_RESPONSES), (60, ADD_MULTIRACK_JOIN_SERVICE_STATUS), (59, INSTANCE_ATTACHED_SUBNETS_MAP), (58, BYTE_WRAPPER_0_2_BUMP), @@ -1605,14 +1606,55 @@ pub trait SledAgentApi { #[endpoint { method = GET, path = "/trust-quorum/coordinator-status", + versions = VERSION_TRUST_QUORUM_NULLABLE_RESPONSES.., }] async fn trust_quorum_coordinator_status( request_context: RequestContext, ) -> Result< - HttpResponseOk>, + HttpResponseOk< + latest::trust_quorum::TrustQuorumCoordinatorStatusResponse, + >, HttpError, >; + /// Get the coordinator status if this node is coordinating a reconfiguration + #[endpoint { + operation_id = "trust_quorum_coordinator_status", + method = GET, + path = "/trust-quorum/coordinator-status", + versions = ..VERSION_TRUST_QUORUM_NULLABLE_RESPONSES, + }] + async fn trust_quorum_coordinator_status_v13( + request_context: RequestContext, + ) -> Result< + HttpResponseOk< + trust_quorum_types_versions::v1::status::CoordinatorStatus, + >, + HttpError, + > { + // These versions were declared as returning an + // Option, but their corresponding OpenAPI documents + // indicated that the CoordinatorStatus response was non-nullable. (This + // is a Dropshot bug; see omicron#11623). We've since changed this API + // to return a non-nullable CoordinatorStatus, making the API match the + // immutable blessed documents. But that leaves open the question of how + // None can be represented on the wire. We return a 404 -- old clients + // will receive it as an error and will handle it reasonably well. + Self::trust_quorum_coordinator_status(request_context).await?.try_map( + |response| { + response.status.ok_or_else(|| { + HttpError::for_client_error( + None, + ClientErrorStatusCode::NOT_FOUND, + "this node is not currently coordinating a trust \ + quorum reconfiguration" + .to_string(), + ) + }) + }, + ) + } + /// Attempt to prepare and commit a trust quorum configuration #[endpoint { method = PUT, @@ -1672,14 +1714,44 @@ pub trait SledAgentApi { #[endpoint { method = GET, path = "/trust-quorum/network-config", + versions = VERSION_TRUST_QUORUM_NULLABLE_RESPONSES.., }] async fn trust_quorum_network_config_get( request_context: RequestContext, ) -> Result< - HttpResponseOk>, + HttpResponseOk, HttpError, >; + /// Get the current network config from trust quorum + #[endpoint { + operation_id = "trust_quorum_network_config_get", + method = GET, + path = "/trust-quorum/network-config", + versions = ..VERSION_TRUST_QUORUM_NULLABLE_RESPONSES, + }] + async fn trust_quorum_network_config_get_v15( + request_context: RequestContext, + ) -> Result< + HttpResponseOk, + HttpError, + > { + // See the comment in trust_quorum_coordinator_status_v13. + Self::trust_quorum_network_config_get(request_context).await?.try_map( + |response| { + response.config.ok_or_else(|| { + HttpError::for_client_error( + None, + ClientErrorStatusCode::NOT_FOUND, + "this node does not have a trust quorum network \ + config" + .to_string(), + ) + }) + }, + ) + } + /// Update the network config in trust quorum #[endpoint { method = PUT, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 90cc020f895..ca3708ce20a 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -63,7 +63,9 @@ use sled_agent_types::support_bundle::{ SupportBundleTransferQueryParams, }; use sled_agent_types::trust_quorum::{ - ProxyCommitRequest, ProxyPrepareAndCommitRequest, TrustQuorumNetworkConfig, + ProxyCommitRequest, ProxyPrepareAndCommitRequest, + TrustQuorumCoordinatorStatusResponse, TrustQuorumNetworkConfig, + TrustQuorumNetworkConfigResponse, }; use sled_agent_types::zone_bundle::{ BundleUtilization, CleanupContext, CleanupContextUpdate, CleanupCount, @@ -75,7 +77,7 @@ use slog_error_chain::InlineErrorChain; use trust_quorum_types::messages::{ CommitRequest, LrtqUpgradeMsg, PrepareAndCommitRequest, ReconfigureMsg, }; -use trust_quorum_types::status::{CommitStatus, CoordinatorStatus, NodeStatus}; +use trust_quorum_types::status::{CommitStatus, NodeStatus}; // Fixed identifiers for prior versions only use sled_agent_types_versions::{v20, v25, v26, v30, v33, v39, v42, v47, v48}; @@ -1550,7 +1552,8 @@ impl SledAgentApi for SledAgentImpl { async fn trust_quorum_coordinator_status( request_context: RequestContext, - ) -> Result>, HttpError> { + ) -> Result, HttpError> + { let sa = request_context.context(); sa.latencies() .instrument_dropshot_handler(&request_context, async { @@ -1562,7 +1565,9 @@ impl SledAgentApi for SledAgentImpl { ) }, )?; - Ok(HttpResponseOk(status)) + Ok(HttpResponseOk(TrustQuorumCoordinatorStatusResponse { + status, + })) }) .await } @@ -1689,7 +1694,7 @@ impl SledAgentApi for SledAgentImpl { async fn trust_quorum_network_config_get( request_context: RequestContext, - ) -> Result>, HttpError> + ) -> Result, HttpError> { let sa = request_context.context(); sa.latencies() @@ -1700,7 +1705,9 @@ impl SledAgentApi for SledAgentImpl { InlineErrorChain::new(&e).to_string(), ) })?; - Ok(HttpResponseOk(config.map(TrustQuorumNetworkConfig::from))) + Ok(HttpResponseOk(TrustQuorumNetworkConfigResponse { + config: config.map(TrustQuorumNetworkConfig::from), + })) }) .await } diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 4e208f7bb82..b9300752032 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -73,7 +73,9 @@ use sled_agent_types::support_bundle::{ SupportBundleTransferQueryParams, }; use sled_agent_types::trust_quorum::{ - ProxyCommitRequest, ProxyPrepareAndCommitRequest, TrustQuorumNetworkConfig, + ProxyCommitRequest, ProxyPrepareAndCommitRequest, + TrustQuorumCoordinatorStatusResponse, TrustQuorumNetworkConfig, + TrustQuorumNetworkConfigResponse, }; use sled_agent_types::zone_bundle::{ BundleUtilization, CleanupContext, CleanupContextUpdate, CleanupCount, @@ -96,7 +98,6 @@ use trust_quorum_types::messages::{ CommitRequest, LrtqUpgradeMsg, PrepareAndCommitRequest, ReconfigureMsg, }; use trust_quorum_types::status::CommitStatus; -use trust_quorum_types::status::CoordinatorStatus; use trust_quorum_types::status::NodeStatus; use super::sled_agent::SledAgent; @@ -1035,7 +1036,8 @@ impl SledAgentApi for SledAgentSimImpl { async fn trust_quorum_coordinator_status( _request_context: RequestContext, - ) -> Result>, HttpError> { + ) -> Result, HttpError> + { method_unimplemented() } @@ -1075,7 +1077,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn trust_quorum_network_config_get( _request_context: RequestContext, - ) -> Result>, HttpError> + ) -> Result, HttpError> { method_unimplemented() } diff --git a/sled-agent/types/versions/src/latest.rs b/sled-agent/types/versions/src/latest.rs index bf9ba540d6a..bfe52f73df7 100644 --- a/sled-agent/types/versions/src/latest.rs +++ b/sled-agent/types/versions/src/latest.rs @@ -294,6 +294,9 @@ pub mod trust_quorum { pub use crate::v13::trust_quorum::ProxyPrepareAndCommitRequest; pub use crate::v15::trust_quorum::TrustQuorumNetworkConfig; + + pub use crate::v61::trust_quorum::TrustQuorumCoordinatorStatusResponse; + pub use crate::v61::trust_quorum::TrustQuorumNetworkConfigResponse; } pub mod uplink { diff --git a/sled-agent/types/versions/src/lib.rs b/sled-agent/types/versions/src/lib.rs index a8be2740538..77757436844 100644 --- a/sled-agent/types/versions/src/lib.rs +++ b/sled-agent/types/versions/src/lib.rs @@ -123,6 +123,8 @@ pub mod v59; pub mod v6; #[path = "add_multirack_join_service_status/mod.rs"] pub mod v60; +#[path = "trust_quorum_nullable_responses/mod.rs"] +pub mod v61; #[path = "multicast_support/mod.rs"] pub mod v7; #[path = "delegate_zvol_to_propolis/mod.rs"] diff --git a/sled-agent/types/versions/src/trust_quorum_nullable_responses/mod.rs b/sled-agent/types/versions/src/trust_quorum_nullable_responses/mod.rs new file mode 100644 index 00000000000..87be0a0225b --- /dev/null +++ b/sled-agent/types/versions/src/trust_quorum_nullable_responses/mod.rs @@ -0,0 +1,11 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `TRUST_QUORUM_NULLABLE_RESPONSES` of the Sled Agent API. +//! +//! Part of [omicron#11623]. See that issue for more details. +//! +//! [omicron#11623]: https://github.com/oxidecomputer/omicron/issues/11623 + +pub mod trust_quorum; diff --git a/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs b/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs new file mode 100644 index 00000000000..c9b724b4c66 --- /dev/null +++ b/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use trust_quorum_types_versions::v1::status::CoordinatorStatus; + +use crate::v15::trust_quorum::TrustQuorumNetworkConfig; + +/// Response type for `trust_quorum_coordinator_status`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TrustQuorumCoordinatorStatusResponse { + /// The coordinator status, or `None` if this node is not currently + /// coordinating a reconfiguration. + pub status: Option, +} + +/// Response type for `trust_quorum_network_config`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct TrustQuorumNetworkConfigResponse { + /// The trust quorum network config, or `None` if a config has not been + /// loaded or received since Sled Agent started. + pub config: Option, +} From 7e1df4fac334e0920cec0320bb18dc7d86fa377d Mon Sep 17 00:00:00 2001 From: Rain Date: Fri, 30 Oct 2026 10:46:42 -0700 Subject: [PATCH 2/2] add the latest epoch --- clients/sled-agent-client/src/lib.rs | 1 + .../src/app/background/tasks/trust_quorum.rs | 28 ++++-- ...968.json => sled-agent-61.0.0-72ccb2.json} | 54 +++++++++-- openapi/sled-agent/sled-agent-latest.json | 2 +- sled-agent/api/src/lib.rs | 25 +++-- sled-agent/src/http_entrypoints.rs | 36 +++++--- .../trust_quorum.rs | 22 ++++- trust-quorum/src/lib.rs | 4 +- trust-quorum/src/task.rs | 92 ++++++++++++++++--- 9 files changed, 208 insertions(+), 56 deletions(-) rename openapi/sled-agent/{sled-agent-61.0.0-9c9968.json => sled-agent-61.0.0-72ccb2.json} (99%) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 9a8c9a4018c..bfd34eaaa27 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -124,6 +124,7 @@ progenitor::generate_api!( SwitchSlot = sled_agent_types_versions::latest::early_networking::SwitchSlot, SystemNetworkingConfig = sled_agent_types_versions::latest::system_networking::SystemNetworkingConfig, Threshold = trust_quorum_types::types::Threshold, + TrustQuorumCoordinatorStatusResponse = sled_agent_types_versions::latest::trust_quorum::TrustQuorumCoordinatorStatusResponse, TxEqConfig = sled_agent_types_versions::latest::early_networking::TxEqConfig, UplinkAddressConfig = sled_agent_types_versions::latest::early_networking::UplinkAddressConfig, VmmRuntimeState = sled_agent_types_versions::latest::instance::VmmRuntimeState, diff --git a/nexus/src/app/background/tasks/trust_quorum.rs b/nexus/src/app/background/tasks/trust_quorum.rs index c1ac4e5938d..c0e4b9c6ef3 100644 --- a/nexus/src/app/background/tasks/trust_quorum.rs +++ b/nexus/src/app/background/tasks/trust_quorum.rs @@ -33,6 +33,7 @@ use sled_agent_client::types::StartSledAgentRequestBody; use sled_agent_client::types::{ ProxyCommitRequest, ProxyPrepareAndCommitRequest, }; +use sled_agent_types::trust_quorum::TrustQuorumCoordinatorStatusResponse; use sled_hardware_types::BaseboardId; use slog::{Logger, error, info, o}; use slog_error_chain::InlineErrorChain; @@ -334,7 +335,7 @@ async fn prepare( )); }; - let status = coordinator_sa + let response = coordinator_sa .trust_quorum_coordinator_status() .await .with_context(|| { @@ -343,15 +344,24 @@ async fn prepare( config.coordinator ) })? - .into_inner() - .status - .ok_or_else(|| { - anyhow!( + .into_inner(); + let status = match response { + TrustQuorumCoordinatorStatusResponse::Coordinating { status } => status, + TrustQuorumCoordinatorStatusResponse::NotCoordinating { + latest_config_epoch, + } => { + let latest_config_epoch = match latest_config_epoch { + Some(epoch) => epoch.to_string(), + None => "none".to_string(), + }; + bail!( "Coordinator {} reported no coordinator status: it is not \ - currently coordinating a reconfiguration.", - config.coordinator - ) - })?; + currently coordinating a reconfiguration (latest persisted \ + configuration epoch: {latest_config_epoch}).", + config.coordinator, + ); + } + }; // Write state back to DB let state = datastore diff --git a/openapi/sled-agent/sled-agent-61.0.0-9c9968.json b/openapi/sled-agent/sled-agent-61.0.0-72ccb2.json similarity index 99% rename from openapi/sled-agent/sled-agent-61.0.0-9c9968.json rename to openapi/sled-agent/sled-agent-61.0.0-72ccb2.json index b8e26023d66..0f66f757078 100644 --- a/openapi/sled-agent/sled-agent-61.0.0-9c9968.json +++ b/openapi/sled-agent/sled-agent-61.0.0-72ccb2.json @@ -10424,18 +10424,54 @@ }, "TrustQuorumCoordinatorStatusResponse": { "description": "Response type for `trust_quorum_coordinator_status`.", - "type": "object", - "properties": { - "status": { - "nullable": true, - "description": "The coordinator status, or `None` if this node is not currently coordinating a reconfiguration.", - "allOf": [ - { - "$ref": "#/components/schemas/CoordinatorStatus" + "oneOf": [ + { + "description": "The node is currently coordinating a reconfiguration or LRTQ upgrade.", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "coordinating" + ] + }, + "status": { + "description": "The coordinator status.", + "allOf": [ + { + "$ref": "#/components/schemas/CoordinatorStatus" + } + ] } + }, + "required": [ + "state", + "status" + ] + }, + { + "description": "The node is not currently coordinating a reconfiguration or LRTQ upgrade.", + "type": "object", + "properties": { + "latest_config_epoch": { + "nullable": true, + "description": "The latest configuration epoch this node has persisted.\n\nIf this is greater than or equal to the epoch that Nexus is driving, the coordinator started that epoch and then stopped for some reason, and Nexus can abort this configuration. If lower than the Nexus epoch, then reconfiguration has not started yet.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "state": { + "type": "string", + "enum": [ + "not_coordinating" + ] + } + }, + "required": [ + "state" ] } - } + ] }, "TrustQuorumNetworkConfig": { "description": "Network configuration used to bring up the control plane.\n\nThis type mirrors `bootstore::schemes::v0::NetworkConfig` but adds `JsonSchema` for API compatibility.", diff --git a/openapi/sled-agent/sled-agent-latest.json b/openapi/sled-agent/sled-agent-latest.json index 9e4ea7eb247..ec5cc6cd68a 120000 --- a/openapi/sled-agent/sled-agent-latest.json +++ b/openapi/sled-agent/sled-agent-latest.json @@ -1 +1 @@ -sled-agent-61.0.0-9c9968.json \ No newline at end of file +sled-agent-61.0.0-72ccb2.json \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 88d829f8007..9074ba8a634 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -1640,17 +1640,22 @@ pub trait SledAgentApi { // immutable blessed documents. But that leaves open the question of how // None can be represented on the wire. We return a 404 -- old clients // will receive it as an error and will handle it reasonably well. + use latest::trust_quorum::TrustQuorumCoordinatorStatusResponse; + Self::trust_quorum_coordinator_status(request_context).await?.try_map( - |response| { - response.status.ok_or_else(|| { - HttpError::for_client_error( - None, - ClientErrorStatusCode::NOT_FOUND, - "this node is not currently coordinating a trust \ - quorum reconfiguration" - .to_string(), - ) - }) + |response| match response { + TrustQuorumCoordinatorStatusResponse::Coordinating { + status, + } => Ok(status), + TrustQuorumCoordinatorStatusResponse::NotCoordinating { + latest_config_epoch: _, + } => Err(HttpError::for_client_error( + None, + ClientErrorStatusCode::NOT_FOUND, + "this node is not currently coordinating a trust \ + quorum reconfiguration" + .to_string(), + )), }, ) } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index ca3708ce20a..13577176f49 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -74,6 +74,7 @@ use sled_agent_types::zone_bundle::{ }; use sled_hardware_types::BaseboardId; use slog_error_chain::InlineErrorChain; +use trust_quorum::CoordinatorStatusSnapshot; use trust_quorum_types::messages::{ CommitRequest, LrtqUpgradeMsg, PrepareAndCommitRequest, ReconfigureMsg, }; @@ -1557,17 +1558,30 @@ impl SledAgentApi for SledAgentImpl { let sa = request_context.context(); sa.latencies() .instrument_dropshot_handler(&request_context, async { - let status = - sa.trust_quorum().coordinator_status().await.map_err( - |e| { - HttpError::for_internal_error( - InlineErrorChain::new(&e).to_string(), - ) - }, - )?; - Ok(HttpResponseOk(TrustQuorumCoordinatorStatusResponse { - status, - })) + let snapshot = sa + .trust_quorum() + .coordinator_status_snapshot() + .await + .map_err(|e| { + HttpError::for_internal_error( + InlineErrorChain::new(&e).to_string(), + ) + })?; + let response = match snapshot { + CoordinatorStatusSnapshot::Coordinating { status } => { + TrustQuorumCoordinatorStatusResponse::Coordinating { + status, + } + } + CoordinatorStatusSnapshot::NotCoordinating { + latest_config_epoch, + } => { + TrustQuorumCoordinatorStatusResponse::NotCoordinating { + latest_config_epoch, + } + } + }; + Ok(HttpResponseOk(response)) }) .await } diff --git a/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs b/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs index c9b724b4c66..93e03fadcff 100644 --- a/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs +++ b/sled-agent/types/versions/src/trust_quorum_nullable_responses/trust_quorum.rs @@ -5,15 +5,29 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use trust_quorum_types_versions::v1::status::CoordinatorStatus; +use trust_quorum_types_versions::v1::types::Epoch; use crate::v15::trust_quorum::TrustQuorumNetworkConfig; /// Response type for `trust_quorum_coordinator_status`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -pub struct TrustQuorumCoordinatorStatusResponse { - /// The coordinator status, or `None` if this node is not currently - /// coordinating a reconfiguration. - pub status: Option, +#[serde(tag = "state", rename_all = "snake_case")] +pub enum TrustQuorumCoordinatorStatusResponse { + /// The node is currently coordinating a reconfiguration or LRTQ upgrade. + Coordinating { + /// The coordinator status. + status: CoordinatorStatus, + }, + /// The node is not currently coordinating a reconfiguration or LRTQ upgrade. + NotCoordinating { + /// The latest configuration epoch this node has persisted. + /// + /// If this is greater than or equal to the epoch that Nexus is driving, + /// the coordinator started that epoch and then stopped for some reason, + /// and Nexus can abort this configuration. If lower than the Nexus + /// epoch, then reconfiguration has not started yet. + latest_config_epoch: Option, + }, } /// Response type for `trust_quorum_network_config`. diff --git a/trust-quorum/src/lib.rs b/trust-quorum/src/lib.rs index 97f9c5df77a..e8dcfb6b379 100644 --- a/trust-quorum/src/lib.rs +++ b/trust-quorum/src/lib.rs @@ -16,4 +16,6 @@ pub use connection_manager::platform_id_to_baseboard_id; pub(crate) use connection_manager::{ ConnToMainMsg, ConnToMainMsgInner, MainToConnMsg, WireMsg, }; -pub use task::{Config, NodeApiError, NodeTask, NodeTaskHandle}; +pub use task::{ + Config, CoordinatorStatusSnapshot, NodeApiError, NodeTask, NodeTaskHandle, +}; diff --git a/trust-quorum/src/task.rs b/trust-quorum/src/task.rs index b828a15e482..e5cae56f7d0 100644 --- a/trust-quorum/src/task.rs +++ b/trust-quorum/src/task.rs @@ -50,6 +50,12 @@ const API_CHANNEL_BOUND: usize = 32; /// similar to using an unbounded channel but will not use all possible memory. const CONN_TO_MAIN_CHANNEL_BOUND: usize = 1024; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CoordinatorStatusSnapshot { + Coordinating { status: CoordinatorStatus }, + NotCoordinating { latest_config_epoch: Option }, +} + #[derive(Debug, Clone)] pub struct Config { pub baseboard_id: BaseboardId, @@ -74,7 +80,7 @@ pub enum NodeApiRequest { ConnMgrStatus { tx: oneshot::Sender }, /// Return the status of this node if it is a coordinator - CoordinatorStatus { tx: oneshot::Sender> }, + CoordinatorStatus { tx: oneshot::Sender }, /// Load a rack secret for the given epoch /// @@ -256,11 +262,28 @@ impl NodeTaskHandle { /// Return the status of this node if it is coordinating the `Prepare` phase /// of a reconfiguration or LRTQ upgrade. Return `Ok(None)` or an error /// otherwise. - /// - /// Must be accessible via a sled-agent API endpoint pub async fn coordinator_status( &self, ) -> Result, NodeApiError> { + // TODO: Clean up coordinator_status after switching all callers to + // coordinator_status_snapshot. + match self.coordinator_status_snapshot().await? { + CoordinatorStatusSnapshot::Coordinating { status } => { + Ok(Some(status)) + } + CoordinatorStatusSnapshot::NotCoordinating { + latest_config_epoch: _, + } => Ok(None), + } + } + + /// Return the coordinator status of this node, including the latest config + /// epoch if this node is not coordinating. + /// + /// Must be accessible via a sled-agent API endpoint. + pub async fn coordinator_status_snapshot( + &self, + ) -> Result { let (tx, rx) = oneshot::channel(); self.tx.send(NodeApiRequest::CoordinatorStatus { tx }).await?; let res = rx.await?; @@ -712,13 +735,22 @@ impl NodeTask { let _ = tx.send(self.conn_mgr.status()); } NodeApiRequest::CoordinatorStatus { tx } => { - let status = self.node.get_coordinator_state().map(|cs| { - CoordinatorStatus { - config: cs.config().clone(), - acked_prepares: cs.op().acked_prepares(), - } - }); - let _ = tx.send(status); + let snapshot = match self.node.get_coordinator_state() { + Some(cs) => CoordinatorStatusSnapshot::Coordinating { + status: CoordinatorStatus { + config: cs.config().clone(), + acked_prepares: cs.op().acked_prepares(), + }, + }, + None => CoordinatorStatusSnapshot::NotCoordinating { + latest_config_epoch: self + .ctx + .persistent_state() + .latest_config() + .map(|c| c.epoch), + }, + }; + let _ = tx.send(snapshot); } NodeApiRequest::LoadRackSecret { epoch, tx } => { let res = self.node.load_rack_secret(&mut self.ctx, epoch); @@ -1479,7 +1511,36 @@ mod tests { } let coordinator = setup.node_handles.first().unwrap(); - coordinator.reconfigure(initial_config).await.unwrap(); + + // Before reconfiguration starts, the coordinator is neither + // coordinating nor has persisted any config. + assert_eq!( + coordinator.coordinator_status_snapshot().await.unwrap(), + CoordinatorStatusSnapshot::NotCoordinating { + latest_config_epoch: None, + }, + ); + + coordinator.reconfigure(initial_config.clone()).await.unwrap(); + + let snapshot = coordinator.coordinator_status_snapshot().await.unwrap(); + let CoordinatorStatusSnapshot::Coordinating { status } = snapshot + else { + panic!("expected coordinator to be coordinating, got {snapshot:?}"); + }; + assert_eq!(status.config.epoch, Epoch(1)); + + // Starting reconfiguration with an identical ReconfigureMsg is rejected + // by `check_epoch`. + assert_matches!( + coordinator.reconfigure(initial_config).await, + Err(NodeApiError::Reconfigure( + ReconfigurationError::PreparedEpochMismatch { + existing: Epoch(1), + new: Epoch(1), + } + )) + ); let poll_interval = Duration::from_millis(10); let poll_max = Duration::from_secs(10); @@ -1515,6 +1576,15 @@ mod tests { } assert_eq!(acked, num_nodes); + // After committing, the coordinator stops coordinating but still + // reports the persisted epoch. + assert_eq!( + coordinator.coordinator_status_snapshot().await.unwrap(), + CoordinatorStatusSnapshot::NotCoordinating { + latest_config_epoch: Some(Epoch(1)) + }, + ); + // Now load the rack secret at all nodes setup .wait_for_rack_secrets_and_assert_equality(