Skip to content

Gate sled-agent startup on trust quorum unlock #11684

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
andrewjstone wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Gate sled-agent startup on trust quorum unlock #11684

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions openapi/bootstrap-agent-lockstep.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,31 @@
}
}
}
},
"/sled-agent-start-status": {
"get": {
"summary": "Get the status of this sled's attempt to start its sled agent.",
"description": "On a cold boot a sled will not start its sled agent (and will therefore not join the underlay) until trust quorum can reconstruct the rack secret. This endpoint is served on the bootstrap network, and is the only way to distinguish a sled that is waiting for unlock from one that has been expunged.",
"operationId": "sled_agent_start_status",
"responses": {
"200": {
"description": "successful operation",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SledAgentStartStatus"
}
}
}
},
"4XX": {
"$ref": "#/components/responses/Error"
},
"5XX": {
"$ref": "#/components/responses/Error"
}
}
}
}
},
"components": {
Expand Down Expand Up @@ -4064,6 +4089,117 @@
}
]
},
"SledAgentStartStatus": {
"description": "The status of starting sled agent",
"oneOf": [
{
"type": "object",
"properties": {
"state": {
"type": "string",
"enum": [
"not_started"
]
}
},
"required": [
"state"
]
},
{
"type": "object",
"properties": {
"connected_peers": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BaseboardId"
},
"uniqueItems": true
},
"state": {
"type": "string",
"enum": [
"waiting_for_trust_quorum_config"
]
}
},
"required": [
"connected_peers",
"state"
]
},
{
"type": "object",
"properties": {
"connected_peers": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BaseboardId"
},
"uniqueItems": true
},
"latest_committed_epoch": {
"type": "integer",
"format": "uint64",
"minimum": 0
},
"state": {
"type": "string",
"enum": [
"waiting_for_rack_unlock"
]
}
},
"required": [
"connected_peers",
"latest_committed_epoch",
"state"
]
},
{
"type": "object",
"properties": {
"epoch": {
"type": "integer",
"format": "uint64",
"minimum": 0
},
"from": {
"$ref": "#/components/schemas/BaseboardId"
},
"state": {
"type": "string",
"enum": [
"expunged"
]
}
},
"required": [
"epoch",
"from",
"state"
]
},
{
"type": "object",
"properties": {
"sled_id": {
"$ref": "#/components/schemas/SledUuid"
},
"state": {
"type": "string",
"enum": [
"started"
]
}
},
"required": [
"sled_id",
"state"
]
}
]
},
"SledUuid": {
"x-rust-type": {
"crate": "omicron-uuid-kinds",
Expand Down
16 changes: 16 additions & 0 deletions sled-agent/bootstrap-agent-lockstep-api/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use bootstrap_agent_lockstep_types::MultirackJoinServiceState;
use bootstrap_agent_lockstep_types::RackInitializeRequest;
use bootstrap_agent_lockstep_types::RackOperationStatus;
use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig;
use bootstrap_agent_lockstep_types::SledAgentStartStatus;
use bootstrap_agent_lockstep_types::scrimlet_reconcilers;
use dropshot::{HttpError, HttpResponseOk, RequestContext, TypedBody};
use omicron_uuid_kinds::MultirackJoinUuid;
Expand Down Expand Up @@ -85,6 +86,21 @@ pub trait BootstrapAgentLockstepApi {
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<BaseboardIds>, HttpError>;

/// Get the status of this sled's attempt to start its sled agent.
///
/// On a cold boot a sled will not start its sled agent (and will therefore
/// not join the underlay) until trust quorum can reconstruct the rack
/// secret. This endpoint is served on the bootstrap network, and is the
/// only way to distinguish a sled that is waiting for unlock from one that
/// has been expunged.
#[endpoint {
method = GET,
path = "/sled-agent-start-status"
}]
async fn sled_agent_start_status(
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<SledAgentStartStatus>, HttpError>;

/// Configure the rack to join a multirack regional cluster
#[endpoint {
method = POST,
Expand Down
21 changes: 21 additions & 0 deletions sled-agent/bootstrap-agent-lockstep-types/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,27 @@ pub struct BaseboardIds {
pub data: IdOrdMap<BootstrapIpOfBaseboardId>,
}

/// The status of starting sled agent
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case", tag = "state")]
pub enum SledAgentStartStatus {
NotStarted,
WaitingForTrustQuorumConfig {
connected_peers: BTreeSet<BaseboardId>,
},
WaitingForRackUnlock {
connected_peers: BTreeSet<BaseboardId>,
latest_committed_epoch: Epoch,
},
Expunged {
epoch: Epoch,
from: BaseboardId,
},
Started {
sled_id: SledUuid,
},
}

/// The state of the commit phase of the trust quorum protocol
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
pub struct CommitState {
Expand Down
11 changes: 11 additions & 0 deletions sled-agent/src/bootstrap/http_entrypoints_lockstep.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use bootstrap_agent_lockstep_types::RackInitializeRequest;
use bootstrap_agent_lockstep_types::RackOperationStatus;
use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig;
use bootstrap_agent_lockstep_types::ReplicatedNetworkConfigContents;
use bootstrap_agent_lockstep_types::SledAgentStartStatus;
use bootstrap_agent_lockstep_types::scrimlet_reconcilers::ScrimletReconcilersStatus;
use dropshot::ClientErrorStatusCode;
use dropshot::{
Expand All @@ -40,6 +41,7 @@ use sled_agent_scrimlet_reconcilers::ScrimletReconcilers;
use slog::Logger;
use sprockets_tls::keys::SprocketsConfig;
use std::sync::Arc;
use tokio::sync::watch;

use crate::bootstrap::rack_ops::RssAccess;
use crate::bootstrap::rack_ops::RssConflictError;
Expand All @@ -55,6 +57,7 @@ pub(crate) struct BootstrapServerContext {
pub(crate) trust_quorum_handle: trust_quorum::NodeTaskHandle,
pub(crate) measurements: Arc<MeasurementsHandle>,
pub(crate) scrimlet_reconcilers: Arc<ScrimletReconcilers>,
pub(crate) sled_agent_start_status: watch::Receiver<SledAgentStartStatus>,
}

impl From<&BootstrapServerContext> for RssContext {
Expand Down Expand Up @@ -202,6 +205,14 @@ impl BootstrapAgentLockstepApi for BootstrapAgentLockstepImpl {
Ok(HttpResponseOk(BaseboardIds { data }))
}

async fn sled_agent_start_status(
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<SledAgentStartStatus>, HttpError> {
let ctx = rqctx.context();
let status = ctx.sled_agent_start_status.borrow().clone();
Ok(HttpResponseOk(status))
}

async fn multirack_join(
rqctx: RequestContext<Self::Context>,
body: TypedBody<MultirackJoinRequest>,
Expand Down
1 change: 1 addition & 0 deletions sled-agent/src/bootstrap/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ mod maghemite;
mod pre_server;
mod pumpkind;
mod rack_ops;
pub(crate) mod rack_unlock;
pub(crate) mod rss_handle;
pub mod secret_retriever;
pub mod server;
Expand Down
Loading
Loading