Embed sush in sled-agent

Cargo.toml

@@ -846,8 +846,8 @@ slog-term = "2.9.1"
 smf = "0.2"
 socket2 = { version = "0.5", features = ["all"] }
 sp-sim = { path = "sp-sim" }
-sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" }
-sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" }
+sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false }
+sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb" }
 sqlformat = "0.3.5"
 sqlparser = { version = "0.61.0", features = [ "visitor" ] }
 static_assertions = "1.1.0"
@@ -861,6 +861,9 @@ subtle = "2.6.1"
 supports-color = "3.0.2"
 support-bundle-collection = { path = "support-bundle-collection" }
 support-bundle-viewer = "0.1.2"
+sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" }
+sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" }
+sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b", features = ["embedded"] }
 swrite = "0.1.0"
 sync-ptr = "0.1.4"
 libsw = { version = "3.5.0", features = ["tokio"] }
@@ -1160,20 +1163,25 @@ branch = "oxide/omicron"
 [patch.crates-io.omicron-workspace-hack]
 path = "workspace-hack"
 
-# Several crates such as crucible and propolis have have a Git dependency on
+# Several crates such as crucible and propolis have a Git dependency on
 # this repo. Omicron itself depends on these crates, which can lead to two
-# copies of these crates in the dependency graph. (As a Git dependency, and as
-# a path dependency.) The goal of omicron-uuid-kinds and
+# copies of these crates in the dependency graph. (As a Git dependency,
+# and as a path dependency.) The goal of omicron-uuid-kinds and
 # omicron-generation-kinds is to provide unified registries of UUID and
 # generation kinds. Two copies of the same kinds floating around is unnecessary
 # and painful.
 #
-# These directives ensure that whenever we see omicron-uuid-kinds or
-# omicron-generation-kinds as a Git dependency, we'll use the path dependency
-# version of the crate instead.
+# Similarly, sush has a Git dependency on this repo's sled-hardware-types,
+# and the BaseboardId in its API must be the same type sled-agent derives
+# from HardwareManager::baseboard().
+#
+# These directives ensure that whenever we see omicron-uuid-kinds,
+# omicron-generation-kinds, or sled-hardware-types as a Git dependency,
+# we'll use the path dependency version of the crate instead.
 #
 # See also: uuid-kinds/README.adoc and generation-kinds/README.adoc.
 [patch."https://github.com/oxidecomputer/omicron"]
 omicron-common = { path = "common" }
 omicron-generation-kinds = { path = "generation-kinds" }
 omicron-uuid-kinds = { path = "uuid-kinds" }
+sled-hardware-types = { path = "sled-hardware/types" }

clients/ddm-admin-client/src/lib.rs

@@ -14,6 +14,8 @@ use either::Either;
 use omicron_common::address::BOOTSTRAP_PREFIX;
 use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH;
 use omicron_common::address::DDMD_PORT;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::SLED_PREFIX_LENGTH;
 use oxnet::Ipv6Net;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
@@ -103,6 +105,29 @@ impl Client {
         self.inner.enable_stats(request).await.map(|resp| resp.into_inner())
     }
 
+    /// Returns every non-bootstrap /64 prefix ddmd has learned. These
+    /// are not all sled subnets: an internal DNS zone is reachable at
+    /// its own reserved /64, advertised by the sled hosting it, and
+    /// RFD 63 reserves others.
+    pub async fn derive_underlay_subnets_from_prefixes(
+        &self,
+    ) -> Result<
+        impl Iterator<Item = Ipv6Subnet<SLED_PREFIX_LENGTH>> + use<>,
+        DdmError,
+    > {
+        let prefixes = self.inner.get_prefixes().await?.into_inner();
+        Ok(prefixes.into_values().flatten().filter_map(|prefix| {
+            let addr = prefix.destination.addr();
+            if prefix.destination.width() == SLED_PREFIX_LENGTH
+                && addr.segments()[0] != BOOTSTRAP_PREFIX
+            {
+                Some(Ipv6Subnet::new(addr))
+            } else {
+                None
+            }
+        }))
+    }
+
     /// Returns the addresses of connected sleds.
     ///
     /// Note: These sleds have not yet been verified.

common/src/address.rs

@@ -232,6 +232,8 @@ pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346;
 pub const BOOTSTORE_PORT: u16 = 12347;
 pub const REPO_DEPOT_PORT: u16 = 12348;
 pub const TRUST_QUORUM_PORT: u16 = 12349;
+pub const SUSH_API_PORT: u16 = 12350;
+pub const SUSH_GOSSIP_PORT: u16 = 12351;
 
 pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080;
 

dev-tools/ls-apis/tests/api_dependencies.out

@@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client)
     consumed by: lldpd (lldp/lldpd) via 1 path
     consumed by: mgd (maghemite/mgd) via 1 path
     consumed by: omicron-nexus (omicron/nexus) via 6 paths
-    consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path
+    consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths
     consumed by: wicketd (omicron/wicketd) via 3 paths
 
 Wicketd Installinator (client: installinator-client)

sled-agent/Cargo.toml

@@ -35,6 +35,8 @@ dropshot.workspace = true
 flate2.workspace = true
 flume.workspace = true
 futures.workspace = true
+gateway-client.workspace = true
+gateway-types.workspace = true
 glob.workspace = true
 hex.workspace = true
 http.workspace = true
@@ -106,8 +108,11 @@ slog-dtrace.workspace = true
 slog-error-chain.workspace = true
 slog-term.workspace = true
 smf.workspace = true
-sprockets-tls.workspace = true
+sprockets-tls = { workspace = true, features = ["ipcc"] }
 strum.workspace = true
+sush-api.workspace = true
+sush-common.workspace = true
+sush-server.workspace = true
 tar.workspace = true
 thiserror.workspace = true
 tofino.workspace = true

sled-agent/config-reconciler/src/handle.rs

@@ -20,6 +20,7 @@ use sled_storage::nested_dataset::NestedDatasetConfig;
 use sled_storage::nested_dataset::NestedDatasetListOptions;
 use sled_storage::nested_dataset::NestedDatasetLocation;
 use slog::Logger;
+use slog::warn;
 use std::collections::HashSet;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -645,6 +646,31 @@ impl AvailableDatasetsReceiver {
         }
     }
 
+    /// Wait until the set of available datasets may have changed.
+    ///
+    /// Returns immediately if the set has changed since this receiver last
+    /// observed it. The test variants' datasets never change, so for them this
+    /// never returns.
+    pub async fn changed(&mut self, log: &Logger) {
+        match &mut self.inner {
+            AvailableDatasetsReceiverInner::Real(receiver) => {
+                if receiver.changed().await.is_err() {
+                    warn!(
+                        log,
+                        "config reconciler task is gone; the set of available \
+                         datasets will never change again",
+                    );
+                    std::future::pending().await
+                }
+            }
+            #[cfg(feature = "testing")]
+            AvailableDatasetsReceiverInner::FakeTempDir { .. }
+            | AvailableDatasetsReceiverInner::FakeStatic(_) => {
+                std::future::pending().await
+            }
+        }
+    }
+
     pub fn all_mounted_debug_datasets(&self) -> Vec<PathInPool> {
         match &self.inner {
             AvailableDatasetsReceiverInner::Real(receiver) => {

sled-agent/src/bootstrap/bootstore_setup.rs

@@ -11,6 +11,7 @@ use super::server::StartError;
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use omicron_common::address::BOOTSTORE_PORT;
+use omicron_common::address::SUSH_GOSSIP_PORT;
 use omicron_common::address::TRUST_QUORUM_PORT;
 use omicron_ddm_admin_client::Client as DdmAdminClient;
 use sled_hardware_types::Baseboard;
@@ -22,6 +23,7 @@ use std::collections::BTreeSet;
 use std::net::Ipv6Addr;
 use std::net::SocketAddrV6;
 use std::time::Duration;
+use tokio::sync::watch;
 
 const BOOTSTORE_FSM_STATE_FILE: &str = "bootstore-fsm-state.json";
 const BOOTSTORE_NETWORK_CONFIG_FILE: &str = "bootstore-network-config.json";
@@ -75,10 +77,14 @@ fn bootstore_network_config_paths(
     Ok(paths)
 }
 
-pub async fn poll_ddmd_for_bootstore_and_tq_peer_update(
+/// Poll ddmd for the set of peer sleds on the bootstrap network, and
+/// publish it to the three consumers that each need it at their own
+/// port: the bootstore, the trust quorum node, and sush gossip.
+pub async fn poll_ddmd_for_peer_updates(
     log: Logger,
     bootstore_node_handle: bootstore::NodeHandle,
     trust_quorum_handle: trust_quorum::NodeTaskHandle,
+    sush_gossip_tx: watch::Sender<BTreeSet<SocketAddrV6>>,
 ) {
     let mut current_bootstore_peers: BTreeSet<SocketAddrV6> = BTreeSet::new();
     let mut current_tq_peers: BTreeSet<SocketAddrV6> = BTreeSet::new();
@@ -137,6 +143,18 @@ pub async fn poll_ddmd_for_bootstore_and_tq_peer_update(
                         return;
                     }
                 }
+                // Inform sush gossip of all known peer addresses.
+                let sush_peers: BTreeSet<_> = addrs
+                    .iter()
+                    .map(|ip| SocketAddrV6::new(*ip, SUSH_GOSSIP_PORT, 0, 0))
+                    .collect();
+                sush_gossip_tx.send_if_modified(|current| {
+                    let changed = *current != sush_peers;
+                    if changed {
+                        *current = sush_peers;
+                    }
+                    changed
+                });
             }
             Err(err) => {
                 warn!(

sled-agent/src/config.rs

@@ -120,6 +120,70 @@ pub struct Config {
     /// root certificates and whether to use local certificate chain or
     /// one over IPCC
     pub sprockets: SprocketsConfig,
+
+    /// Settings for the Support Shell server (RFD 620). If this is absent, no
+    /// Support Shell server runs on this sled.
+    #[serde(default)]
+    pub sush: Option<SushConfig>,
+}
+
+/// Configuration for the Support Shell (`sush`) server that runs in the global
+/// zone. See RFD 620.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SushConfig {
+    /// PEM files holding the trusted root certificates for job requests,
+    /// one certificate per file. A job whose signature does not chain to
+    /// one of these is refused, so an empty list means no job could ever
+    /// run, and config parsing rejects it. To run a sled with no Support
+    /// Shell server, omit the `[sush]` section instead.
+    #[serde(deserialize_with = "SushConfig::nonempty_roots")]
+    pub roots: Vec<Utf8PathBuf>,
+
+    /// Where to record job output before an encrypted dataset is available.
+    /// Must be on a ramdisk so that it is never persisted unencrypted.
+    #[serde(default = "SushConfig::default_ramdisk_dir")]
+    pub ramdisk_dir: Utf8PathBuf,
+
+    /// Maximum output a single job may record before an encrypted dataset is
+    /// available. The default is small because the ramdisk is global zone
+    /// memory.
+    #[serde(default = "SushConfig::default_ramdisk_max_output_mb")]
+    pub ramdisk_max_output_mb: u32,
+
+    /// Maximum output a single job may record once an encrypted dataset is
+    /// available.
+    #[serde(default = "SushConfig::default_max_output_mb")]
+    pub max_output_mb: u32,
+}
+
+impl SushConfig {
+    // TODO: check that this is the right default. `/var/run/oxide` is tmpfs
+    // and is created during bootstrap, but it exists to hold ZFS key files
+    // (see `illumos_utils::zfs::KEYPATH_ROOT`), not bulk job output.
+    fn default_ramdisk_dir() -> Utf8PathBuf {
+        "/var/run/oxide/sush".into()
+    }
+
+    fn default_ramdisk_max_output_mb() -> u32 {
+        64
+    }
+
+    fn default_max_output_mb() -> u32 {
+        10 * 1024
+    }
+
+    fn nonempty_roots<'de, D: serde::Deserializer<'de>>(
+        deserializer: D,
+    ) -> Result<Vec<Utf8PathBuf>, D::Error> {
+        let roots = Vec::deserialize(deserializer)?;
+        if roots.is_empty() {
+            return Err(serde::de::Error::custom(
+                "sush needs at least one root certificate to run",
+            ));
+        }
+        Ok(roots)
+    }
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -210,4 +274,37 @@ mod test {
         }
         assert!(configs_seen > 0, "No sled-agent configs found");
     }
+
+    #[test]
+    fn test_sush_config() {
+        // Only `roots` is required. The rest have defaults.
+        let sush: SushConfig =
+            toml::from_str(r#"roots = ["/pkg/sush-root.pem"]"#).unwrap();
+        assert_eq!(sush.roots, vec![Utf8PathBuf::from("/pkg/sush-root.pem")]);
+        assert_eq!(sush.ramdisk_dir, SushConfig::default_ramdisk_dir());
+        assert_eq!(
+            sush.ramdisk_max_output_mb,
+            SushConfig::default_ramdisk_max_output_mb()
+        );
+        assert_eq!(sush.max_output_mb, SushConfig::default_max_output_mb());
+
+        let sush: SushConfig = toml::from_str(
+            r#"
+            roots = ["/pkg/sush-root.pem"]
+            ramdisk_dir = "/var/run/sush"
+            ramdisk_max_output_mb = 8
+            max_output_mb = 128
+            "#,
+        )
+        .unwrap();
+        assert_eq!(sush.ramdisk_dir, "/var/run/sush");
+        assert_eq!(sush.ramdisk_max_output_mb, 8);
+        assert_eq!(sush.max_output_mb, 128);
+
+        // No roots would mean no jobs can ever run, so reject at parse time.
+        let err = toml::from_str::<SushConfig>("roots = []").unwrap_err();
+        assert!(
+            err.to_string().contains("needs at least one root certificate")
+        );
+    }
 }

sled-agent/src/lib.rs

@@ -35,6 +35,7 @@ pub mod server;
 pub mod services;
 mod sled_agent;
 mod support_bundle;
+pub mod sush;
 mod swap_device;
 mod vmm_reservoir;
 mod zone_bundle;