From 01f98ba88b7421b44011364c90e658f53f17eb70 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sun, 6 Sep 2026 15:02:36 -0600 Subject: [PATCH 01/38] Embed sush in sled-agent This begins the integration of the [Support Shell](https://github.com/oxidecomputer/sush) ([RFD 620](https://rfd.shared.oxide.computer/rfd/0620)) service into the wider control plane, starting with sled-agent. Sled-agent now builds the sush job manager during bootstrap as a long-running task and serves its HTTP API on the bootstrap network from the start. Once the sled has an underlay address, it serves the same API there, too. In addition to its client-facing API, sush also gossips sessions, jobs, and events over sprockets on the bootstrap network using [rumors](https://github.com/oxidecomputer/rumors). Gossip peers are discovered using the bootstore's existing DDM polling loop, and a map polled from MGS resolves job targets given as cubby numbers. The records sush must trust across reboots (its gossip identity and its execution boundary) live on the M.2s' cluster datasets. Job output is recorded on the ramdisk until an encrypted debug dataset is mounted, then moves there with a raised size limit. The signatures on all jobs must chain to one of the configured certs; the cert here is a sandbox root which **must be replaced with a production cert before shipping** (see oxidecomputer/sush#46). Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 376 ++++++++++++++- Cargo.toml | 24 +- clients/ddm-admin-client/src/lib.rs | 21 + common/src/address.rs | 2 + dev-tools/ls-apis/tests/api_dependencies.out | 2 +- sled-agent/Cargo.toml | 7 +- sled-agent/config-reconciler/src/handle.rs | 22 + sled-agent/src/bootstrap/bootstore_setup.rs | 17 +- sled-agent/src/config.rs | 78 ++++ sled-agent/src/lib.rs | 1 + sled-agent/src/long_running_tasks.rs | 49 +- sled-agent/src/server.rs | 30 +- sled-agent/src/sush.rs | 427 ++++++++++++++++++ smf/sled-agent/gimlet-standalone/config.toml | 5 + .../gimlet-standalone/sush-sandbox.pem | 11 + smf/sled-agent/gimlet/config.toml | 5 + smf/sled-agent/gimlet/sush-sandbox.pem | 11 + smf/sled-agent/non-gimlet/config.toml | 5 + smf/sled-agent/non-gimlet/sush-sandbox.pem | 11 + workspace-hack/Cargo.toml | 40 +- 20 files changed, 1098 insertions(+), 46 deletions(-) create mode 100644 sled-agent/src/sush.rs create mode 100644 smf/sled-agent/gimlet-standalone/sush-sandbox.pem create mode 100644 smf/sled-agent/gimlet/sush-sandbox.pem create mode 100644 smf/sled-agent/non-gimlet/sush-sandbox.pem diff --git a/Cargo.lock b/Cargo.lock index d65c05264e3..69566a16986 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -656,6 +656,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "before" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "bytes", + "dashu-int", + "dsi-bitstream", + "serde", + "serde_json", + "static_assertions", + "suanpan", + "thiserror 2.0.18", +] + [[package]] name = "bhyve_api" version = "0.0.0" @@ -1024,6 +1039,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + [[package]] name = "bzip2" version = "0.4.4" @@ -2571,6 +2592,25 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dashu-base" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64b04cdfc4c8533100fe00304eb9687173bda47b1f1dac8af12ba13712ed49d" + +[[package]] +name = "dashu-int" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ee98721d5d223e5b64b642dd9588b79d9ef415554b13720308b77d628c3be6" +dependencies = [ + "cfg-if", + "dashu-base", + "num-modular", + "rustversion", + "static_assertions", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -3440,6 +3480,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dsi-bitstream" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f3e8a7ab0f1148cf7a792807d330f1e417dda3f4f122526e609a83925b4823" +dependencies = [ + "num-primitive", +] + [[package]] name = "dsl_auto_type" version = "0.2.0" @@ -4759,6 +4808,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "headers" @@ -5104,6 +5158,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -6748,6 +6808,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +dependencies = [ + "hashbrown 0.17.0", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -8568,6 +8637,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-modular" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8e500409e6cd603b03e477c26a6caecdc27ac58979a53e881c75eafc079f44" + +[[package]] +name = "num-primitive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0178502a58a2514f927965f80216c0ea2533b5364061544ea8e09b79343d4183" + [[package]] name = "num-rational" version = "0.4.2" @@ -9702,6 +9783,8 @@ dependencies = [ "fmd-adm", "fmd-adm-sys", "futures", + "gateway-client", + "gateway-types", "glob", "guppy", "hex", @@ -9785,6 +9868,9 @@ dependencies = [ "sprockets-tls", "strum 0.27.2", "subprocess", + "sush-api", + "sush-common", + "sush-server", "tar", "test-strategy", "thiserror 2.0.18", @@ -9879,6 +9965,7 @@ dependencies = [ "base64ct", "bitflags 1.3.2", "bitflags 2.11.0", + "blake3", "bstr", "buf-list", "byte-wrapper", @@ -9895,6 +9982,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "crossterm 0.28.1", + "crypto-bigint 0.5.5", "crypto-common 0.1.7", "crypto-common 0.2.2", "ctutils", @@ -9904,6 +9992,7 @@ dependencies = [ "der 0.7.10", "derive_more 2.1.1", "derive_more-impl", + "dice-verifier 0.3.0-pre0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", "digest 0.10.7", "digest 0.11.3", "dof 0.3.0", @@ -9931,6 +10020,7 @@ dependencies = [ "group 0.13.0", "hashbrown 0.15.5", "hashbrown 0.16.1", + "hashbrown 0.17.0", "heck 0.4.1", "hex", "hickory-proto 0.25.2", @@ -9950,6 +10040,7 @@ dependencies = [ "lalrpop-util", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "linux-raw-sys 0.4.15", "log", "managed", @@ -10007,6 +10098,7 @@ dependencies = [ "smallvec 1.15.1", "spin", "spki 0.7.3", + "sprockets-tls", "string_cache", "strum 0.26.3", "strum 0.27.2", @@ -10786,6 +10878,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + [[package]] name = "p256" version = "0.14.0" @@ -10825,6 +10929,20 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "rand_core 0.6.4", + "sha2 0.10.9", +] + [[package]] name = "p521" version = "0.14.0" @@ -12089,6 +12207,16 @@ dependencies = [ "psl-types", ] +[[package]] +name = "pwd" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "qorb" version = "0.4.1" @@ -12393,7 +12521,7 @@ dependencies = [ "indoc 2.0.7", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum 0.26.3", "unicode-segmentation", @@ -12874,6 +13002,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "ron" version = "0.8.1" @@ -12978,6 +13115,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rumors" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "async-stream", + "before", + "blake3", + "bytes", + "ciborium", + "futures", + "futures-util", + "hex", + "itertools 0.14.0", + "rand 0.8.6", + "seq-macro", + "serde", + "smallvec 1.15.1", + "static_assertions", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tokio-stream", +] + [[package]] name = "russh" version = "0.62.4" @@ -13018,9 +13180,9 @@ dependencies = [ "ml-kem", "module-lattice", "num-bigint", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "pageant", "pbkdf2", "pkcs1 0.8.0-rc.4", @@ -13040,8 +13202,8 @@ dependencies = [ "sha3 0.12.0", "signature 3.0.0", "spki 0.8.0", - "ssh-encoding", - "ssh-key", + "ssh-encoding 0.3.0", + "ssh-key 0.7.0-rc.11", "subtle", "thiserror 2.0.18", "tokio", @@ -13058,7 +13220,7 @@ checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" dependencies = [ "log", "nix 0.31.2", - "ssh-encoding", + "ssh-encoding 0.3.0", "windows-sys 0.61.2", ] @@ -15021,7 +15183,7 @@ checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" [[package]] name = "sprockets-tls" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "anyhow", "attest-data 0.5.0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", @@ -15054,7 +15216,7 @@ dependencies = [ [[package]] name = "sprockets-tls-test-utils" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "camino", "pki-playground 0.2.0 (git+https://github.com/oxidecomputer/pki-playground?rev=7600756029ce046a02c6234aa84ce230cc5eaa04)", @@ -15092,6 +15254,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ssh-cipher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" +dependencies = [ + "cipher 0.4.4", + "ssh-encoding 0.2.0", +] + [[package]] name = "ssh-cipher" version = "0.3.0" @@ -15106,10 +15278,21 @@ dependencies = [ "ctutils", "des", "poly1305 0.9.0", - "ssh-encoding", + "ssh-encoding 0.3.0", "zeroize", ] +[[package]] +name = "ssh-encoding" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" +dependencies = [ + "base64ct", + "pem-rfc7468 0.7.0", + "sha2 0.10.9", +] + [[package]] name = "ssh-encoding" version = "0.3.0" @@ -15125,6 +15308,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ssh-key" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" +dependencies = [ + "ed25519-dalek 2.2.0", + "p256 0.13.2", + "p384 0.13.1", + "p521 0.13.3", + "rand_core 0.6.4", + "rsa 0.9.10", + "sec1 0.7.3", + "serde", + "sha2 0.10.9", + "signature 2.2.0", + "ssh-cipher 0.2.0", + "ssh-encoding 0.2.0", + "subtle", + "zeroize", +] + [[package]] name = "ssh-key" version = "0.7.0-rc.11" @@ -15137,17 +15342,17 @@ dependencies = [ "ed25519-dalek 3.0.0", "hex", "hmac 0.13.0", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "rand_core 0.10.0", "rsa 0.10.0-rc.18", "sec1 0.8.1", "sha1 0.11.0", "sha2 0.11.0", "signature 3.0.0", - "ssh-cipher", - "ssh-encoding", + "ssh-cipher 0.3.0", + "ssh-encoding 0.3.0", "zeroize", ] @@ -15328,6 +15533,14 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "suanpan" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=be1d76947f75f27e7f53be438a93d350019dd026#be1d76947f75f27e7f53be438a93d350019dd026" +dependencies = [ + "dashu-int", +] + [[package]] name = "subprocess" version = "0.2.15" @@ -15447,6 +15660,110 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" +[[package]] +name = "sush-api" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "chrono", + "dropshot", + "http-range-header", + "hyper", + "schemars 0.8.22", + "serde", + "sled-hardware-types", + "sush-common", + "thiserror 1.0.69", +] + +[[package]] +name = "sush-common" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "bytes", + "bytesize", + "chrono", + "crypto-bigint 0.5.5", + "ed25519-dalek 2.2.0", + "hex", + "http", + "http-range-header", + "p256 0.13.2", + "pem-rfc7468 0.7.0", + "rand 0.8.6", + "rand_core 0.6.4", + "rlimit", + "rustix 1.1.4", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sha3 0.10.8", + "signature 2.2.0", + "sled-hardware-types", + "slog", + "ssh-key 0.6.7", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite 0.28.0", + "x509-cert", +] + +[[package]] +name = "sush-server" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +dependencies = [ + "async-trait", + "atomicwrites", + "bytes", + "bytesize", + "camino", + "chrono", + "ciborium", + "clap", + "dropshot", + "ed25519-dalek 2.2.0", + "futures", + "http", + "http-body-util", + "http-range-header", + "hyper", + "hyper-util", + "libc", + "lru 0.18.3", + "memmap2", + "p256 0.13.2", + "percent-encoding", + "pwd", + "qorb", + "rand_core 0.6.4", + "rumors", + "rustix 1.1.4", + "rustls", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sled-hardware-types", + "slog", + "slog-term", + "sprockets-tls", + "ssh-key 0.6.7", + "sush-api", + "sush-common", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-tungstenite 0.28.0", + "tokio-util", + "x509-cert", +] + [[package]] name = "swrite" version = "0.1.0" @@ -15903,9 +16220,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -16110,6 +16427,18 @@ dependencies = [ "tungstenite 0.23.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.28.0", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -16894,6 +17223,23 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1 0.10.6", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "twox-hash" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index 304268defd3..bdd036dd8e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -846,8 +846,8 @@ slog-term = "2.9.1" smf = "0.2" socket2 = { version = "0.5", features = ["all"] } sp-sim = { path = "sp-sim" } -sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" } -sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "a233079e04d9688454486c452b694587a28c5257" } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false } +sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb" } sqlformat = "0.3.5" sqlparser = { version = "0.61.0", features = [ "visitor" ] } static_assertions = "1.1.0" @@ -861,6 +861,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } @@ -1160,20 +1163,25 @@ branch = "oxide/omicron" [patch.crates-io.omicron-workspace-hack] path = "workspace-hack" -# Several crates such as crucible and propolis have have a Git dependency on +# Several crates such as crucible and propolis have a Git dependency on # this repo. Omicron itself depends on these crates, which can lead to two -# copies of these crates in the dependency graph. (As a Git dependency, and as -# a path dependency.) The goal of omicron-uuid-kinds and +# copies of these crates in the dependency graph. (As a Git dependency, +# and as a path dependency.) The goal of omicron-uuid-kinds and # omicron-generation-kinds is to provide unified registries of UUID and # generation kinds. Two copies of the same kinds floating around is unnecessary # and painful. # -# These directives ensure that whenever we see omicron-uuid-kinds or -# omicron-generation-kinds as a Git dependency, we'll use the path dependency -# version of the crate instead. +# Similarly, sush has a Git dependency on this repo's sled-hardware-types, +# and the BaseboardId in its API must be the same type sled-agent derives +# from HardwareManager::baseboard(). +# +# These directives ensure that whenever we see omicron-uuid-kinds, +# omicron-generation-kinds, or sled-hardware-types as a Git dependency, +# we'll use the path dependency version of the crate instead. # # See also: uuid-kinds/README.adoc and generation-kinds/README.adoc. [patch."https://github.com/oxidecomputer/omicron"] omicron-common = { path = "common" } omicron-generation-kinds = { path = "generation-kinds" } omicron-uuid-kinds = { path = "uuid-kinds" } +sled-hardware-types = { path = "sled-hardware/types" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index c9450ecefaa..462ba47ebea 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -14,6 +14,8 @@ use either::Either; use omicron_common::address::BOOTSTRAP_PREFIX; use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH; use omicron_common::address::DDMD_PORT; +use omicron_common::address::Ipv6Subnet; +use omicron_common::address::SLED_PREFIX_LENGTH; use oxnet::Ipv6Net; use sled_hardware_types::underlay::BootstrapInterface; use slog::Logger; @@ -103,6 +105,25 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } + /// Returns the underlay subnets DDM advertises. Callers must probe + /// each subnet for what they expect to find, because sleds also + /// advertise internal DNS subnets, and RFD 63 reserves a services + /// prefix. + pub async fn derive_underlay_subnets_from_prefixes( + &self, + ) -> Result< + impl Iterator> + use<>, + DdmError, + > { + let prefixes = self.inner.get_prefixes().await?.into_inner(); + Ok(prefixes.into_values().flatten().filter_map(|prefix| { + let addr = prefix.destination.addr(); + (prefix.destination.width() == SLED_PREFIX_LENGTH + && addr.segments()[0] != BOOTSTRAP_PREFIX) + .then(|| Ipv6Subnet::new(addr)) + })) + } + /// Returns the addresses of connected sleds. /// /// Note: These sleds have not yet been verified. diff --git a/common/src/address.rs b/common/src/address.rs index 953ccd40df2..d8c36c0f196 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -232,6 +232,8 @@ pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346; pub const BOOTSTORE_PORT: u16 = 12347; pub const REPO_DEPOT_PORT: u16 = 12348; pub const TRUST_QUORUM_PORT: u16 = 12349; +pub const SUSH_API_PORT: u16 = 12350; +pub const SUSH_GOSSIP_PORT: u16 = 12351; pub const BOOTSTRAP_AGENT_LOCKSTEP_PORT: u16 = 8080; diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 637c0a97484..60c7a46bd59 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -49,7 +49,7 @@ Management Gateway Service (client: gateway-client) consumed by: lldpd (lldp/lldpd) via 1 path consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 6 paths - consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 2 paths consumed by: wicketd (omicron/wicketd) via 3 paths Wicketd Installinator (client: installinator-client) diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 47577484c32..3514f8dd2c9 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -35,6 +35,8 @@ dropshot.workspace = true flate2.workspace = true flume.workspace = true futures.workspace = true +gateway-client.workspace = true +gateway-types.workspace = true glob.workspace = true hex.workspace = true http.workspace = true @@ -106,8 +108,11 @@ slog-dtrace.workspace = true slog-error-chain.workspace = true slog-term.workspace = true smf.workspace = true -sprockets-tls.workspace = true +sprockets-tls = { workspace = true, features = ["ipcc"] } strum.workspace = true +sush-api.workspace = true +sush-common.workspace = true +sush-server.workspace = true tar.workspace = true thiserror.workspace = true tofino.workspace = true diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index d37ea02360f..947f30bfef6 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -645,6 +645,28 @@ impl AvailableDatasetsReceiver { } } + /// Wait until the set of available datasets may have changed. + /// + /// Returns immediately if the set has changed since this receiver last + /// observed it. The test variants' datasets never change, so for them this + /// never returns. + pub async fn changed(&mut self) { + match &mut self.inner { + AvailableDatasetsReceiverInner::Real(receiver) => { + // An error means the reconciler task is gone, in which case the + // set can never change again. + if receiver.changed().await.is_err() { + std::future::pending().await + } + } + #[cfg(feature = "testing")] + AvailableDatasetsReceiverInner::FakeTempDir { .. } + | AvailableDatasetsReceiverInner::FakeStatic(_) => { + std::future::pending().await + } + } + } + pub fn all_mounted_debug_datasets(&self) -> Vec { match &self.inner { AvailableDatasetsReceiverInner::Real(receiver) => { diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs index 020f7506761..dd4508feaf9 100644 --- a/sled-agent/src/bootstrap/bootstore_setup.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -11,6 +11,7 @@ use super::server::StartError; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use omicron_common::address::BOOTSTORE_PORT; +use omicron_common::address::SUSH_GOSSIP_PORT; use omicron_common::address::TRUST_QUORUM_PORT; use omicron_ddm_admin_client::Client as DdmAdminClient; use sled_hardware_types::Baseboard; @@ -22,6 +23,7 @@ use std::collections::BTreeSet; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::time::Duration; +use tokio::sync::watch; const BOOTSTORE_FSM_STATE_FILE: &str = "bootstore-fsm-state.json"; const BOOTSTORE_NETWORK_CONFIG_FILE: &str = "bootstore-network-config.json"; @@ -75,10 +77,11 @@ fn bootstore_network_config_paths( Ok(paths) } -pub async fn poll_ddmd_for_bootstore_and_tq_peer_update( +pub async fn poll_ddmd_for_peer_updates( log: Logger, bootstore_node_handle: bootstore::NodeHandle, trust_quorum_handle: trust_quorum::NodeTaskHandle, + sush_gossip_tx: watch::Sender>, ) { let mut current_bootstore_peers: BTreeSet = BTreeSet::new(); let mut current_tq_peers: BTreeSet = BTreeSet::new(); @@ -137,6 +140,18 @@ pub async fn poll_ddmd_for_bootstore_and_tq_peer_update( return; } } + // Inform sush gossip of all known peer addresses. + let sush_peers: BTreeSet<_> = addrs + .iter() + .map(|ip| SocketAddrV6::new(*ip, SUSH_GOSSIP_PORT, 0, 0)) + .collect(); + sush_gossip_tx.send_if_modified(|current| { + let changed = *current != sush_peers; + if changed { + *current = sush_peers; + } + changed + }); } Err(err) => { warn!( diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 5f4d2d83be3..af4683292bb 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -120,6 +120,56 @@ pub struct Config { /// root certificates and whether to use local certificate chain or /// one over IPCC pub sprockets: SprocketsConfig, + + /// Settings for the Support Shell server (RFD 620). If this is absent, no + /// Support Shell server runs on this sled. + #[serde(default)] + pub sush: Option, +} + +/// Configuration for the Support Shell (`sush`) server that runs in the global +/// zone. See RFD 620. +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SushConfig { + /// PEM files holding the trusted root certificates for job requests, + /// one certificate per file. A job whose signature does not chain to + /// one of these is refused, so an empty list means no job can ever run. + pub roots: Vec, + + /// Where to record job output before an encrypted dataset is available. + /// This is on the ramdisk, so it costs global zone memory and does not + /// survive a reboot. + #[serde(default = "SushConfig::default_ramdisk_dir")] + pub ramdisk_dir: Utf8PathBuf, + + /// Maximum output a single job may record before an encrypted dataset is + /// available. The default is small because the ramdisk is global zone + /// memory. + #[serde(default = "SushConfig::default_ramdisk_max_output_mb")] + pub ramdisk_max_output_mb: u32, + + /// Maximum output a single job may record once an encrypted dataset is + /// available. + #[serde(default = "SushConfig::default_max_output_mb")] + pub max_output_mb: u32, +} + +impl SushConfig { + // TODO: check that this is the right default. `/var/run/oxide` is tmpfs + // and is created during bootstrap, but it exists to hold ZFS key files + // (see `illumos_utils::zfs::KEYPATH_ROOT`), not bulk job output. + fn default_ramdisk_dir() -> Utf8PathBuf { + "/var/run/oxide/sush".into() + } + + fn default_ramdisk_max_output_mb() -> u32 { + 64 + } + + fn default_max_output_mb() -> u32 { + 10 * 1024 + } } #[derive(Debug, thiserror::Error)] @@ -210,4 +260,32 @@ mod test { } assert!(configs_seen > 0, "No sled-agent configs found"); } + + #[test] + fn test_sush_config() { + // Only `roots` is required. The rest have defaults. + let sush: SushConfig = + toml::from_str(r#"roots = ["/pkg/sush-root.pem"]"#).unwrap(); + assert_eq!(sush.roots, vec![Utf8PathBuf::from("/pkg/sush-root.pem")]); + assert_eq!(sush.ramdisk_dir, SushConfig::default_ramdisk_dir()); + assert_eq!( + sush.ramdisk_max_output_mb, + SushConfig::default_ramdisk_max_output_mb() + ); + assert_eq!(sush.max_output_mb, SushConfig::default_max_output_mb()); + + let sush: SushConfig = toml::from_str( + r#" + roots = [] + ramdisk_dir = "/var/run/sush" + ramdisk_max_output_mb = 8 + max_output_mb = 128 + "#, + ) + .unwrap(); + assert!(sush.roots.is_empty()); + assert_eq!(sush.ramdisk_dir, "/var/run/sush"); + assert_eq!(sush.ramdisk_max_output_mb, 8); + assert_eq!(sush.max_output_mb, 128); + } } diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index a30026d0134..f41bc1d4949 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -35,6 +35,7 @@ pub mod server; pub mod services; mod sled_agent; mod support_bundle; +pub mod sush; mod swap_device; mod vmm_reservoir; mod zone_bundle; diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index bb34420cdb2..c0b74859bb9 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -14,7 +14,7 @@ use crate::artifact_store::{ArtifactStore, SledAgentArtifactStoreWrapper}; use crate::bootstrap::bootstore_setup::{ - new_bootstore_config, poll_ddmd_for_bootstore_and_tq_peer_update, + new_bootstore_config, poll_ddmd_for_peer_updates, }; use crate::bootstrap::secret_retriever::{ ConfigurableSecretRetriever, ConfigurableSecretRetrieverHandle, @@ -24,6 +24,7 @@ use crate::config::Config; use crate::hardware_monitor::{HardwareMonitor, HardwareMonitorHandle}; use crate::services::ServiceManager; use crate::sled_agent::SledAgent; +use crate::sush::{GossipInputs, SushHandles, spawn_sush_tasks}; use crate::zone_bundle::ZoneBundler; use bootstore::schemes::v0 as bootstore; use key_manager::{KeyManager, StorageKeyRequester}; @@ -41,9 +42,10 @@ use sled_storage::config::MountConfig; use sled_storage::disk::RawSyntheticDisk; use slog::{Logger, info}; use sprockets_tls::keys::SprocketsConfig; -use std::net::Ipv6Addr; +use std::collections::BTreeSet; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; use trust_quorum; /// A mechanism for interacting with all long running tasks that can be shared @@ -91,6 +93,9 @@ pub struct LongRunningTaskHandles { /// Handle to access the set of reference measurements pub measurements: Arc, + + /// Handles to the Support Shell server, if it is running + pub sush: Option, } pub struct LongRunningTaskResult { @@ -196,17 +201,51 @@ pub async fn spawn_all_longrunning_tasks( ) .await; + let (sush_gossip_peers_tx, sush_gossip_peers_rx) = + watch::channel(BTreeSet::new()); let bootstore = spawn_bootstore_tasks( log, &config_reconciler, &hardware_manager, global_zone_bootstrap_ip, trust_quorum.clone(), + sush_gossip_peers_tx, ) .await; let health_monitor = spawn_health_monitor_tasks(log).await; + // sush must work when the control plane doesn't, so it starts here rather + // than waiting for rack membership, and serves its API on the bootstrap + // network. Once this sled is told its underlay address, + // `crate::server::Server::start` serves the same API there too. + let sush = match &config.sush { + Some(sush_config) => { + spawn_sush_tasks( + log, + sush_config, + hardware_manager.baseboard().into(), + config_reconciler + .internal_disks_rx() + .current() + .all_cluster_datasets() + .collect(), + GossipInputs { + sprockets: config.sprockets.clone(), + measurements: measurements.clone(), + bootstrap_ip: global_zone_bootstrap_ip, + peers: sush_gossip_peers_rx, + }, + config_reconciler.available_datasets_rx(), + ) + .await + } + None => { + info!(log, "sush not configured"); + None + } + }; + LongRunningTaskResult { long_running_task_handles: LongRunningTaskHandles { config_reconciler, @@ -221,6 +260,7 @@ pub async fn spawn_all_longrunning_tasks( secret_retriever: secret_retriever_config, artifact_store, measurements, + sush, }, config_reconciler_spawn_token, sled_agent_started_tx, @@ -324,6 +364,7 @@ async fn spawn_bootstore_tasks( hardware_manager: &HardwareManager, global_zone_bootstrap_ip: Ipv6Addr, tq_handle: trust_quorum::NodeTaskHandle, + sush_gossip_tx: watch::Sender>, ) -> bootstore::NodeHandle { let config = new_bootstore_config( &config_reconciler @@ -346,7 +387,7 @@ async fn spawn_bootstore_tasks( let log = log.new(o!("component" => "bootstore_ddmd_poller")); let node_handle2 = node_handle.clone(); tokio::spawn(async move { - poll_ddmd_for_bootstore_and_tq_peer_update(log, node_handle2, tq_handle) + poll_ddmd_for_peer_updates(log, node_handle2, tq_handle, sush_gossip_tx) .await }); diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index c23e7033150..c4538d47c45 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -10,6 +10,7 @@ use super::sled_agent::SledAgent; use crate::long_running_tasks::LongRunningTaskHandles; use crate::nexus::make_nexus_client; use crate::services::ServiceManager; +use crate::sush::SushHandles; use internal_dns_resolver::Resolver; use omicron_uuid_kinds::SledUuid; use sled_agent_config_reconciler::ConfigReconcilerSpawnToken; @@ -18,12 +19,16 @@ use slog::Logger; use slog_error_chain::InlineErrorChain; use std::net::SocketAddr; use std::sync::Arc; +use sush_server::JobManager; /// Packages up a [`SledAgent`], running the sled agent API under a Dropshot /// server wired up to the sled agent pub struct Server { /// Dropshot server for the API. http_server: dropshot::HttpServer, + + /// The Support Shell's task handles and API server, if it is running. + sush: Option<(SushHandles, dropshot::HttpServer>)>, } impl Server { @@ -57,6 +62,7 @@ impl Server { let nexus_client = make_nexus_client(&log, resolver); + let sush_handles = long_running_tasks_handles.sush.clone(); let sled_agent = SledAgent::new( &config, log.clone(), @@ -86,7 +92,20 @@ impl Server { ))) .start() .map_err(|error| format!("initializing server: {}", error))?; - Ok(Server { http_server }) + + // Now that this sled knows its underlay address, sush can serve + // its API there, too. + let sush = sush_handles.and_then(|handles| { + match handles.start_api(*sled_address.ip()) { + Ok(server) => Some((handles, server)), + Err(error) => { + warn!(log, "failed to start sush server"; "error" => error); + None + } + } + }); + + Ok(Server { http_server, sush }) } pub(crate) fn sled_agent(&self) -> &SledAgent { @@ -103,6 +122,13 @@ impl Server { } pub async fn close(self) -> Result<(), String> { - self.http_server.close().await + let sush = match self.sush { + Some((handles, server)) => { + handles.shutdown(); + server.close().await + } + None => Ok(()), + }; + self.http_server.close().await.and(sush) } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs new file mode 100644 index 00000000000..21fc5c440fb --- /dev/null +++ b/sled-agent/src/sush.rs @@ -0,0 +1,427 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The Support Shell (`sush`) server that runs in the global zone. +//! +//! See RFD 620. This server runs jobs targeted at this sled on behalf of +//! Oxide support. A job arrives signed, and this sled runs it only when +//! the signature chains to one of the root certificates in its config. +//! The server is deliberately independent of the rest of the control plane, +//! because it is one of the tools you reach for when the control plane is +//! the thing that is broken. It therefore starts during bootstrap, before +//! this sled knows whether it is part of a rack at all. +//! +//! Startup is in two parts, because its inputs become available at different +//! times: +//! +//! * [`spawn_sush_tasks`] builds the job manager as soon as we know our +//! baseboard, and is called from `long_running_tasks`. It serves the +//! API on the bootstrap network, so the switch zone proxy can reach +//! this sled before RSS runs. +//! * [`SushHandles::start_api`] serves the API again on the underlay, +//! and is called from `server` once this sled has been told its +//! underlay address. +//! +//! Gossip runs over sprockets on the bootstrap network, so jobs and +//! sessions are shared across sleds. A universe is a shared gossip +//! history; peers that meet merge into one by a dominance rule. +//! A restarted sled re-seeds its universe, rejoins the rack's, then +//! replays its history without re-executing it. Its gossip identity +//! is stored in a _bookmark_. +//! +//! The bookmark, like every record sush must trust across reboots, +//! lives in the [sush locker](https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs). +//! Each locker record is one file on each M.2. Stores write every +//! copy, and loads adopt a record only when the copies show it cannot +//! be stale. The locker is not a small bootstore: the bootstore holds +//! rack-wide facts that sleds may recover from their peers, but a locker +//! record says what this sled itself did or committed to do. These +//! guarantees are critical to the correctness of the +//! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, +//! which sush uses to synchronize job and event sets. + +use crate::config::SushConfig; +use camino::Utf8PathBuf; +use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; +use gateway_client::Client as MgsClient; +use gateway_types::component::SpType; +use omicron_common::address::{ + MGS_PORT, SUSH_API_PORT, SUSH_GOSSIP_PORT, get_switch_zone_address, +}; +use omicron_ddm_admin_client::Client as DdmClient; +use sled_agent_config_reconciler::AvailableDatasetsReceiver; +use sled_agent_measurements::MeasurementsHandle; +use sled_hardware_types::BaseboardId; +use slog::{Logger, debug, error, info, o, warn}; +use slog_error_chain::InlineErrorChain; +use sprockets_tls::keys::SprocketsConfig; +use std::collections::BTreeSet; +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; +use std::sync::Arc; +use std::time::Duration; +use tokio::fs::create_dir_all; +use tokio::spawn; +use tokio::sync::watch; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +use sush_common::targets::{Cubbies, MAX_CUBBY}; +use sush_server::executor::PathIsolation; +use sush_server::gossip::{GossipConfig, isolated, lonely, spawn_gossip}; +use sush_server::link::CorpusSource; +use sush_server::locker::Locker; +use sush_server::output::{JobOutputDir, OutputDirs}; +use sush_server::server::ApiServer; +use sush_server::{JobManager, seed_gossip}; + +/// Subdirectory of an encrypted dataset that job output is recorded in. +const SUSH_OUTPUT_SUBDIR: &str = "sush"; + +/// How often to refresh the cubby map from MGS. +const MGS_POLL_INTERVAL: Duration = Duration::from_secs(30); + +/// How long to wait for an MGS candidate to answer. +const MGS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Maximum size of a request body the API will accept. The largest thing a +/// client sends is a signed job request or a certificate, both small. +const REQUEST_MAX_BODY_BYTES: usize = 0xFFFF; + +/// Handles to the Support Shell server's tasks. +#[derive(Clone)] +pub struct SushHandles { + log: Logger, + manager: Arc, + shutdown: CancellationToken, +} + +impl SushHandles { + /// Start serving the Support Shell API at `ip`. + pub fn start_api( + &self, + ip: Ipv6Addr, + ) -> Result>, String> { + let bind_address = + SocketAddr::V6(SocketAddrV6::new(ip, SUSH_API_PORT, 0, 0)); + let api = sush_api::sush_api_mod::api_description::() + .map_err(|err| format!("failed to describe sush API: {err}"))?; + let server = ServerBuilder::new( + api, + Arc::clone(&self.manager), + self.log.new(o!("component" => "dropshot (sush)")), + ) + .config(ConfigDropshot { + bind_address, + default_request_body_max_bytes: REQUEST_MAX_BODY_BYTES, + // An interactive job holds a websocket open for as long as it runs, + // so a handler must outlive the request that created it. + default_handler_task_mode: HandlerTaskMode::Detached, + log_headers: vec![], + compression: Default::default(), + }) + .start() + .map_err(|err| err.to_string())?; + info!( + self.log, "started sush server"; + "address" => %bind_address, + ); + Ok(server) + } + + /// Stop the Support Shell server's tasks. + pub fn shutdown(&self) { + self.shutdown.cancel(); + } +} + +/// What gossip needs from the sled: its sprockets identity, its reference +/// measurements, the bootstrap address to listen on, and where to find its +/// peers. +pub struct GossipInputs { + pub sprockets: SprocketsConfig, + pub measurements: Arc, + pub bootstrap_ip: Ipv6Addr, + pub peers: watch::Receiver>, +} + +/// Start the Support Shell server's tasks, or log the reason and return +/// `None` when they cannot start. The locker gets one slot on each of +/// `cluster_datasets`. +pub async fn spawn_sush_tasks( + log: &Logger, + config: &SushConfig, + own_baseboard: BaseboardId, + cluster_datasets: Vec, + gossip: GossipInputs, + available_datasets_rx: AvailableDatasetsReceiver, +) -> Option { + let log = log.new(o!("component" => "sush")); + + // Job output starts on the ramdisk, because an encrypted dataset cannot be + // mounted until trust quorum is established, and sush must be useful + // before then. + if let Err(err) = create_dir_all(&config.ramdisk_dir).await { + error!( + log, + "not starting sush server: could not create job output directory"; + "directory" => %config.ramdisk_dir, + "error" => InlineErrorChain::new(&err), + ); + return None; + } + let (output_dirs_tx, output_dirs_rx) = watch::channel(OutputDirs::new( + config.ramdisk_dir.as_std_path(), + mb_to_bytes(config.ramdisk_max_output_mb), + )); + + if config.roots.is_empty() { + warn!(log, "sush has no root certificates, so no job will ever run"); + } + let shutdown = CancellationToken::new(); + + // A slot that cannot be created is still handed to the locker, which + // decides what a failed slot means for each load and store. + let slots: Vec = + cluster_datasets.iter().map(|dataset| dataset.join("sush")).collect(); + for slot in &slots { + if let Err(err) = create_dir_all(slot).await { + warn!( + log, "could not create a locker slot"; + "slot" => %slot, + "error" => InlineErrorChain::new(&err), + ); + } + } + let locker = Locker::new(&log, slots); + + // Gossip re-reads the attestation corpus on every handshake, because a + // software update changes it. A sled that cannot gossip still serves local + // jobs. + let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; + let corpus: CorpusSource = Arc::new({ + let log = log.clone(); + move || match measurements.current_measurements() { + Ok(corpus) => corpus, + Err(e) => { + error!(log, "measurement error"; e); + vec![] + } + } + }); + let listen_addr = SocketAddrV6::new(bootstrap_ip, SUSH_GOSSIP_PORT, 0, 0); + let (universe, linked) = match spawn_gossip( + &log, + GossipConfig::default(), + sprockets, + corpus, + listen_addr, + peers, + seed_gossip(&log, &locker).await, + shutdown.clone(), + ) + .await + { + Ok((_, universe, linked)) => (universe, linked), + Err(err) => { + warn!( + log, + "gossip disabled, this sled serves local jobs only"; + "error" => InlineErrorChain::new(&err), + ); + // This seed never reaches a peer, so it is not worth + // storing. The null locker stores nothing, leaving the + // bookmark on the M.2s untouched for the next boot. + let seed = seed_gossip(&log, &Locker::null()).await; + (isolated(seed.into_rumors()), lonely()) + } + }; + + let (tx_cubbies, rx_cubbies) = watch::channel(Cubbies::new()); + let mut manager = match JobManager::new( + log.clone(), + PathIsolation::Enable, + JobOutputDir::new(output_dirs_rx), + own_baseboard, + rx_cubbies, + universe, + linked, + &locker, + &config.roots, + shutdown.clone(), + ) + .await + { + Ok(manager) => manager, + Err(err) => { + error!(log, "not starting sush server"; "error" => InlineErrorChain::new(&err)); + return None; + } + }; + spawn(poll_mgs_for_cubbies( + log.new(o!("component" => "cubby map")), + tx_cubbies, + )); + + // The state manager runs until shutdown and nothing waits on it, + // so all we can usefully do with its handle is notice when it stops. + if let Some(join) = manager.take_join_handle() { + let log = log.clone(); + spawn(async move { + match join.await { + Ok(()) => info!(log, "sush state manager stopped"), + Err(err) => error!( + log, + "sush state manager failed"; + "error" => InlineErrorChain::new(&err), + ), + } + }); + } + + spawn(promote_output_dir( + log.clone(), + available_datasets_rx, + output_dirs_tx, + mb_to_bytes(config.max_output_mb), + )); + + info!(log, "started sush job manager"); + let handles = SushHandles { log, manager: Arc::new(manager), shutdown }; + + match handles.start_api(bootstrap_ip) { + Ok(server) => { + let shutdown = handles.shutdown.clone(); + spawn(async move { + shutdown.cancelled().await; + server.close().await.ok(); + }); + } + Err(err) => warn!( + handles.log, + "sush is not serving on the bootstrap network"; + "error" => err, + ), + } + + Some(handles) +} + +/// Start recording new job output on an encrypted debug dataset as soon +/// as one is mounted, with a raised size limit. Output already recorded +/// on the ramdisk stays there, readable until reboot. +async fn promote_output_dir( + log: Logger, + mut available_datasets_rx: AvailableDatasetsReceiver, + output_dirs_tx: watch::Sender, + max_output_bytes: u64, +) { + loop { + if let Some(dataset) = + available_datasets_rx.all_mounted_debug_datasets().first() + { + let dir = dataset.path.join(SUSH_OUTPUT_SUBDIR); + match create_dir_all(&dir).await { + Ok(()) => { + output_dirs_tx.send_modify(|dirs| { + *dirs = + dirs.moved_to(dir.as_std_path(), max_output_bytes) + }); + info!( + log, "recording job output on encrypted dataset"; + "directory" => %dir, + ); + return; + } + Err(err) => error!( + log, + "could not create job output directory on encrypted dataset, leaving it on the ramdisk"; + "directory" => %dir, + "error" => InlineErrorChain::new(&err), + ), + } + } + available_datasets_rx.changed().await; + } +} + +fn mb_to_bytes(mb: u32) -> u64 { + u64::from(mb) * 1024 * 1024 +} + +/// Keep the cubby map current from MGS's view of the SPs. A job may +/// name its target sled by cubby, and the map says which baseboard +/// is in each cubby. +/// +/// DDM's advertised subnets are only candidates (see +/// [`DdmClient::derive_underlay_subnets_from_prefixes`]), so we must +/// probe for MGS. Each round's answers merge into the existing map, +/// so a probe outage never erases it. +async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { + let ddm = match DdmClient::localhost(&log) { + Ok(ddm) => ddm, + Err(err) => { + error!( + log, "not polling MGS, cubby-targeted jobs will not run here"; + "error" => InlineErrorChain::new(&err), + ); + return; + } + }; + let client = reqwest::ClientBuilder::new() + .connect_timeout(MGS_PROBE_TIMEOUT) + .timeout(MGS_PROBE_TIMEOUT) + .build() + .expect("failed to build an HTTP client"); + loop { + match ddm.derive_underlay_subnets_from_prefixes().await { + Ok(subnets) => { + let mut map = Cubbies::new(); + for subnet in subnets { + let addr = SocketAddrV6::new( + get_switch_zone_address(subnet), + MGS_PORT, + 0, + 0, + ); + let mgs = MgsClient::new_with_client( + &format!("http://{addr}"), + client.clone(), + log.clone(), + ); + if mgs.sp_local_switch_id().await.is_err() { + continue; + } + for cubby in 0..=MAX_CUBBY { + match mgs.sp_get(&SpType::Sled, cubby.into()).await { + Ok(state) => { + let state = state.into_inner(); + map.insert( + cubby, + BaseboardId { + part_number: state.model, + serial_number: state.serial_number, + }, + ); + } + Err(err) => { + debug!( + log, "no SP state for cubby"; + "cubby" => cubby, "error" => %err, + ); + } + } + } + } + cubbies.send_modify(|current| current.extend(map)); + } + Err(err) => { + warn!( + log, "unable to fetch prefixes"; + "error" => InlineErrorChain::new(&err), + ); + } + } + sleep(MGS_POLL_INTERVAL).await; + } +} diff --git a/smf/sled-agent/gimlet-standalone/config.toml b/smf/sled-agent/gimlet-standalone/config.toml index 547ea950c11..34b46bbd7e4 100644 --- a/smf/sled-agent/gimlet-standalone/config.toml +++ b/smf/sled-agent/gimlet-standalone/config.toml @@ -84,3 +84,8 @@ resolve = { which = "ipcc" } attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/gimlet-standalone/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/config.toml b/smf/sled-agent/gimlet/config.toml index 92a9cda8392..eca0e7ef7a9 100644 --- a/smf/sled-agent/gimlet/config.toml +++ b/smf/sled-agent/gimlet/config.toml @@ -79,3 +79,8 @@ resolve = { which = "ipcc" } attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/gimlet/sush-sandbox.pem b/smf/sled-agent/gimlet/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/gimlet/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index 4f46cd56cb7..3342db3e333 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -129,3 +129,8 @@ resolve = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/test-sprocket attest = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/test-alias-1.key.pem", cert_chain = "/opt/oxide/sled-agent/pkg/test-alias-1.certlist.pem", log = "/opt/oxide/sled-agent/pkg/sprockets-log.bin", test_corpus = [ "/opt/oxide/sled-agent/pkg/testing-measurements/corim-rot.cbor", "/opt/oxide/sled-agent/pkg/testing-measurements/corim-sp.cbor"] } roots = ["/opt/oxide/sled-agent/pkg/test-root-a.cert.pem"] enforce = "Enforced" + +# Support Shell (RFD 620). TODO: replace the sandbox root with the +# production certificate. +[sush] +roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] diff --git a/smf/sled-agent/non-gimlet/sush-sandbox.pem b/smf/sled-agent/non-gimlet/sush-sandbox.pem new file mode 100644 index 00000000000..17b6a761ef2 --- /dev/null +++ b/smf/sled-agent/non-gimlet/sush-sandbox.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 +NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP +eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy +b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg +m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ +ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk +t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= +-----END CERTIFICATE----- diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d9d151cc286..ddcf58c17f6 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -28,6 +28,7 @@ base64 = { version = "0.22.1" } base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } +blake3 = { version = "1.8.5", features = ["mmap", "rayon"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } @@ -40,6 +41,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -48,6 +50,7 @@ daft = { version = "0.1.8", features = ["derive", "newtype-uuid1", "oxnet01", "u data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -71,6 +74,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } hex = { version = "0.4.3", features = ["serde"] } hickory-proto = { version = "0.25.2", features = ["serde", "text-parsing"] } @@ -120,7 +124,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -135,9 +139,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -180,6 +185,7 @@ base64 = { version = "0.22.1" } base64ct = { version = "1.8.3", default-features = false, features = ["std"] } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } +blake3 = { version = "1.8.5", features = ["mmap", "rayon"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } @@ -193,6 +199,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -202,6 +209,7 @@ data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } derive_more-impl = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -225,6 +233,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } heck = { version = "0.4.1", features = ["unicode"] } hex = { version = "0.4.3", features = ["serde"] } @@ -275,7 +284,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -291,9 +300,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -335,13 +345,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-linux-gnu.build-dependencies] @@ -350,13 +361,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.dependencies] @@ -369,7 +381,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.build-dependencies] @@ -382,7 +394,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.dependencies] @@ -395,7 +407,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.build-dependencies] @@ -408,7 +420,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-illumos.dependencies] @@ -426,7 +438,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } @@ -446,7 +458,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } From 552801fb0fce02ff41e7431ba1f447d4fa509689 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:01:36 -0600 Subject: [PATCH 02/38] Log the start of sush tasks Co-Authored-By: Claude Mythos 5 --- sled-agent/src/long_running_tasks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index c0b74859bb9..27549ec91c2 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -221,6 +221,7 @@ pub async fn spawn_all_longrunning_tasks( // `crate::server::Server::start` serves the same API there too. let sush = match &config.sush { Some(sush_config) => { + info!(log, "Starting sush tasks"); spawn_sush_tasks( log, sush_config, From 1b65c0fa80bc66ddc8cdccb662e48b1d6234c075 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:25:43 -0600 Subject: [PATCH 03/38] Fix SushConfig::ramdisk_dir doc Co-Authored-By: Claude Mythos 5 --- sled-agent/src/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index af4683292bb..4b92545cdd1 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -138,8 +138,7 @@ pub struct SushConfig { pub roots: Vec, /// Where to record job output before an encrypted dataset is available. - /// This is on the ramdisk, so it costs global zone memory and does not - /// survive a reboot. + /// Must be on a ramdisk so that it is never persisted unencrypted. #[serde(default = "SushConfig::default_ramdisk_dir")] pub ramdisk_dir: Utf8PathBuf, From 7f810f4c2dae97f0c54c2a7284c0cdd679616308 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:41:24 -0600 Subject: [PATCH 04/38] Fix DDM subnet and cubby map docs Also rephrase a `bool.then()` as `if`. Co-Authored-By: Claude Mythos 5 --- clients/ddm-admin-client/src/lib.rs | 18 +++++++++++------- sled-agent/src/sush.rs | 15 +++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 462ba47ebea..84cee697204 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -105,10 +105,10 @@ impl Client { self.inner.enable_stats(request).await.map(|resp| resp.into_inner()) } - /// Returns the underlay subnets DDM advertises. Callers must probe - /// each subnet for what they expect to find, because sleds also - /// advertise internal DNS subnets, and RFD 63 reserves a services - /// prefix. + /// Returns every non-bootstrap /64 prefix ddmd has learned. These + /// are not all sled subnets: an internal DNS zone is reachable at + /// its own reserved /64, advertised by the sled hosting it, and + /// RFD 63 reserves others. pub async fn derive_underlay_subnets_from_prefixes( &self, ) -> Result< @@ -118,9 +118,13 @@ impl Client { let prefixes = self.inner.get_prefixes().await?.into_inner(); Ok(prefixes.into_values().flatten().filter_map(|prefix| { let addr = prefix.destination.addr(); - (prefix.destination.width() == SLED_PREFIX_LENGTH - && addr.segments()[0] != BOOTSTRAP_PREFIX) - .then(|| Ipv6Subnet::new(addr)) + if prefix.destination.width() == SLED_PREFIX_LENGTH + && addr.segments()[0] != BOOTSTRAP_PREFIX + { + Some(Ipv6Subnet::new(addr)) + } else { + None + } })) } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 21fc5c440fb..2097afeda0e 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -349,14 +349,13 @@ fn mb_to_bytes(mb: u32) -> u64 { u64::from(mb) * 1024 * 1024 } -/// Keep the cubby map current from MGS's view of the SPs. A job may -/// name its target sled by cubby, and the map says which baseboard -/// is in each cubby. -/// -/// DDM's advertised subnets are only candidates (see -/// [`DdmClient::derive_underlay_subnets_from_prefixes`]), so we must -/// probe for MGS. Each round's answers merge into the existing map, -/// so a probe outage never erases it. +/// Periodically ask MGS which baseboard sits in each cubby, and +/// publish the map. A job may name its target sled by cubby; +/// this map resolves it. MGS answers at a fixed address within its +/// switch zone's subnet, so each round we ask at that address in every +/// /64 ddmd has learned, and take answers from any that respond. The +/// answers merge into the map, so a round that goes unanswered never +/// erases it. async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { let ddm = match DdmClient::localhost(&log) { Ok(ddm) => ddm, From d44a8383a4afc73285470ba6e66e32f1e9f79cde Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 18:57:56 -0600 Subject: [PATCH 05/38] Document poll_ddmd_for_peer_updates Co-Authored-By: Claude Mythos 5 --- sled-agent/src/bootstrap/bootstore_setup.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs index dd4508feaf9..e833dcf40b2 100644 --- a/sled-agent/src/bootstrap/bootstore_setup.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -77,6 +77,9 @@ fn bootstore_network_config_paths( Ok(paths) } +/// Poll ddmd for the set of peer sleds on the bootstrap network, and +/// publish it to the three consumers that each need it at their own +/// port: the bootstore, the trust quorum node, and sush gossip. pub async fn poll_ddmd_for_peer_updates( log: Logger, bootstore_node_handle: bootstore::NodeHandle, From 3dc9c683c31b6220ba46b1eead96bd7264e49a63 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:02:15 -0600 Subject: [PATCH 06/38] Use ByteCount for byte counts Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 2097afeda0e..a4b7171844f 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -49,6 +49,7 @@ use gateway_types::component::SpType; use omicron_common::address::{ MGS_PORT, SUSH_API_PORT, SUSH_GOSSIP_PORT, get_switch_zone_address, }; +use omicron_common::api::external::ByteCount; use omicron_ddm_admin_client::Client as DdmClient; use sled_agent_config_reconciler::AvailableDatasetsReceiver; use sled_agent_measurements::MeasurementsHandle; @@ -172,7 +173,7 @@ pub async fn spawn_sush_tasks( } let (output_dirs_tx, output_dirs_rx) = watch::channel(OutputDirs::new( config.ramdisk_dir.as_std_path(), - mb_to_bytes(config.ramdisk_max_output_mb), + ByteCount::from_mebibytes_u32(config.ramdisk_max_output_mb).to_bytes(), )); if config.roots.is_empty() { @@ -283,7 +284,7 @@ pub async fn spawn_sush_tasks( log.clone(), available_datasets_rx, output_dirs_tx, - mb_to_bytes(config.max_output_mb), + ByteCount::from_mebibytes_u32(config.max_output_mb).to_bytes(), )); info!(log, "started sush job manager"); @@ -345,10 +346,6 @@ async fn promote_output_dir( } } -fn mb_to_bytes(mb: u32) -> u64 { - u64::from(mb) * 1024 * 1024 -} - /// Periodically ask MGS which baseboard sits in each cubby, and /// publish the map. A job may name its target sled by cubby; /// this map resolves it. MGS answers at a fixed address within its From ae91da078135a9cbd6506d1bb4dece2872e8e815 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:19:16 -0600 Subject: [PATCH 07/38] Return an anyhow::Result from SushHandles::start_api Co-Authored-By: Claude Mythos 5 --- sled-agent/src/server.rs | 2 +- sled-agent/src/sush.rs | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index c4538d47c45..6b9cd6de56c 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -99,7 +99,7 @@ impl Server { match handles.start_api(*sled_address.ip()) { Ok(server) => Some((handles, server)), Err(error) => { - warn!(log, "failed to start sush server"; "error" => error); + warn!(log, "failed to start sush server"; "error" => #%error); None } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index a4b7171844f..8c625b8a3fe 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -42,6 +42,7 @@ //! which sush uses to synchronize job and event sets. use crate::config::SushConfig; +use anyhow::Context; use camino::Utf8PathBuf; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; use gateway_client::Client as MgsClient; @@ -102,11 +103,11 @@ impl SushHandles { pub fn start_api( &self, ip: Ipv6Addr, - ) -> Result>, String> { + ) -> anyhow::Result>> { let bind_address = SocketAddr::V6(SocketAddrV6::new(ip, SUSH_API_PORT, 0, 0)); let api = sush_api::sush_api_mod::api_description::() - .map_err(|err| format!("failed to describe sush API: {err}"))?; + .context("describing the sush API")?; let server = ServerBuilder::new( api, Arc::clone(&self.manager), @@ -122,7 +123,7 @@ impl SushHandles { compression: Default::default(), }) .start() - .map_err(|err| err.to_string())?; + .context("starting the sush API server")?; info!( self.log, "started sush server"; "address" => %bind_address, @@ -301,7 +302,7 @@ pub async fn spawn_sush_tasks( Err(err) => warn!( handles.log, "sush is not serving on the bootstrap network"; - "error" => err, + "error" => #%err, ), } From 301e9d59ebe7b4a435ddf43f5c740cccd220d8da Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:19:52 -0600 Subject: [PATCH 08/38] Kill useless GossipInputs doc --- sled-agent/src/sush.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 8c625b8a3fe..e172adf04f5 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -137,9 +137,6 @@ impl SushHandles { } } -/// What gossip needs from the sled: its sprockets identity, its reference -/// measurements, the bootstrap address to listen on, and where to find its -/// peers. pub struct GossipInputs { pub sprockets: SprocketsConfig, pub measurements: Arc, From 59f30ff40a119a0a0f1e861af29dd0f63bb3f207 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:27:00 -0600 Subject: [PATCH 09/38] Wrap long log line --- sled-agent/src/sush.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index e172adf04f5..8de9c3a0939 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -334,7 +334,8 @@ async fn promote_output_dir( } Err(err) => error!( log, - "could not create job output directory on encrypted dataset, leaving it on the ramdisk"; + "could not create job output directory on encrypted dataset, \ + leaving it on the ramdisk"; "directory" => %dir, "error" => InlineErrorChain::new(&err), ), From 4e9c78ed6f483cec7aa61791bbb2543e6a840052 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:43:06 -0600 Subject: [PATCH 10/38] Warn if the config reconciler task goes away Co-Authored-By: Claude Mythos 5 --- sled-agent/config-reconciler/src/handle.rs | 10 +++++++--- sled-agent/src/sush.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index 947f30bfef6..3c2f71572e2 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -20,6 +20,7 @@ use sled_storage::nested_dataset::NestedDatasetConfig; use sled_storage::nested_dataset::NestedDatasetListOptions; use sled_storage::nested_dataset::NestedDatasetLocation; use slog::Logger; +use slog::warn; use std::collections::HashSet; use std::sync::Arc; use std::sync::OnceLock; @@ -650,12 +651,15 @@ impl AvailableDatasetsReceiver { /// Returns immediately if the set has changed since this receiver last /// observed it. The test variants' datasets never change, so for them this /// never returns. - pub async fn changed(&mut self) { + pub async fn changed(&mut self, log: &Logger) { match &mut self.inner { AvailableDatasetsReceiverInner::Real(receiver) => { - // An error means the reconciler task is gone, in which case the - // set can never change again. if receiver.changed().await.is_err() { + warn!( + log, + "config reconciler task is gone; the set of available \ + datasets will never change again", + ); std::future::pending().await } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 8de9c3a0939..83192941cbf 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -341,7 +341,7 @@ async fn promote_output_dir( ), } } - available_datasets_rx.changed().await; + available_datasets_rx.changed(&log).await; } } From 5a652270de63002fc3966601f1940230f69b8b72 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:47:14 -0600 Subject: [PATCH 11/38] fix locker link Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 83192941cbf..e759292abac 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -31,7 +31,7 @@ //! is stored in a _bookmark_. //! //! The bookmark, like every record sush must trust across reboots, -//! lives in the [sush locker](https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs). +//! lives in the [sush locker]. //! Each locker record is one file on each M.2. Stores write every //! copy, and loads adopt a record only when the copies show it cannot //! be stale. The locker is not a small bootstore: the bootstore holds @@ -40,6 +40,9 @@ //! guarantees are critical to the correctness of the //! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, //! which sush uses to synchronize job and event sets. +//! +//! [sush locker]: +//! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs use crate::config::SushConfig; use anyhow::Context; From 3c7792c50267455128009e05090fedfd856c6dce Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:48:04 -0600 Subject: [PATCH 12/38] Declaude Co-authored-by: Eliza Weisman --- sled-agent/src/long_running_tasks.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index 27549ec91c2..beca56ac781 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -215,10 +215,11 @@ pub async fn spawn_all_longrunning_tasks( let health_monitor = spawn_health_monitor_tasks(log).await; - // sush must work when the control plane doesn't, so it starts here rather - // than waiting for rack membership, and serves its API on the bootstrap - // network. Once this sled is told its underlay address, - // `crate::server::Server::start` serves the same API there too. + // Because sush must work even when the control plane doesn't, we start it + // before waiting for rack membership. Initially, the sush API is served on + // the bootstrap network. Once the sled is told its underlay address, + // `crate::server::Server::start` will also serve the sush API on the + // bootstrap network. let sush = match &config.sush { Some(sush_config) => { info!(log, "Starting sush tasks"); From b1eefa9dbe7a2683b2b0ab0062aceff19088f4c0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 19:53:50 -0600 Subject: [PATCH 13/38] better corpus closure Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index e759292abac..f0077d44b83 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -197,15 +197,19 @@ pub async fn spawn_sush_tasks( } let locker = Locker::new(&log, slots); - // Gossip re-reads the attestation corpus on every handshake, because a - // software update changes it. A sled that cannot gossip still serves local - // jobs. let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; + + // On every gossip protocol handshake, we must re-read the attestation + // corpus, because a software update may have changed it. This closure + // is invoked in order to do that. let corpus: CorpusSource = Arc::new({ let log = log.clone(); move || match measurements.current_measurements() { Ok(corpus) => corpus, Err(e) => { + // If reading the measurements fails, this sled cannot + // participate in the gossip protocol, but can still serve + // local jobs. error!(log, "measurement error"; e); vec![] } From 4c7ccb146bf135ee8c4ca40ea342ecf3383a6265 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 20:05:52 -0600 Subject: [PATCH 14/38] Fix up applied suggestions Co-Authored-By: Claude Mythos 5 --- sled-agent/src/long_running_tasks.rs | 2 +- sled-agent/src/sush.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index beca56ac781..cbd1671cdba 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -219,7 +219,7 @@ pub async fn spawn_all_longrunning_tasks( // before waiting for rack membership. Initially, the sush API is served on // the bootstrap network. Once the sled is told its underlay address, // `crate::server::Server::start` will also serve the sush API on the - // bootstrap network. + // underlay network. let sush = match &config.sush { Some(sush_config) => { info!(log, "Starting sush tasks"); diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index f0077d44b83..0e5736bee1b 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -198,7 +198,7 @@ pub async fn spawn_sush_tasks( let locker = Locker::new(&log, slots); let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; - + // On every gossip protocol handshake, we must re-read the attestation // corpus, because a software update may have changed it. This closure // is invoked in order to do that. From ecaf96d4b949ab286f6e8c77ae233b702451c4a0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 20:25:36 -0600 Subject: [PATCH 15/38] Improve sush module doc Co-Authored-By: finch Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 43 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 0e5736bee1b..75cc0da40a4 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -23,23 +23,34 @@ //! and is called from `server` once this sled has been told its //! underlay address. //! -//! Gossip runs over sprockets on the bootstrap network, so jobs and -//! sessions are shared across sleds. A universe is a shared gossip -//! history; peers that meet merge into one by a dominance rule. -//! A restarted sled re-seeds its universe, rejoins the rack's, then -//! replays its history without re-executing it. Its gossip identity -//! is stored in a _bookmark_. +//! Gossip runs over sprockets on the bootstrap network. A job may be +//! addressed to a single sled, but the messages carrying jobs, +//! sessions, and events spread to every sled. The set of causally +//! related messages is called a _universe_: peers in the same universe +//! can gossip and converge on its contents; peers in different +//! universes cannot gossip at all. Every identity in a universe +//! descends from a single seed. When sleds meet, a deterministic rule +//! picks whose seed wins, and the others bootstrap from it, each +//! taking a slice of the winner's identity space. A restarted sled +//! therefore rejoins the rack's universe and replays what it missed +//! without re-executing it. Each sled stores its identity in a +//! _bookmark_, so that a restart resumes it rather than growing a new +//! one on every boot. //! -//! The bookmark, like every record sush must trust across reboots, -//! lives in the [sush locker]. -//! Each locker record is one file on each M.2. Stores write every -//! copy, and loads adopt a record only when the copies show it cannot -//! be stale. The locker is not a small bootstore: the bootstore holds -//! rack-wide facts that sleds may recover from their peers, but a locker -//! record says what this sled itself did or committed to do. These -//! guarantees are critical to the correctness of the -//! [rumors](https://github.com/oxidecomputer/rumors) gossip algorithm, -//! which sush uses to synchronize job and event sets. +//! The bookmark, like every record sush must trust across reboots, lives +//! in the [sush locker]. When we write a record to the locker, we actually +//! write two copies, one to each M.2. When we load a record, it only +//! succeeds if both copies are exact matches, or if one of them is +//! entirely missing (the latter to handle the case of M.2 hardware +//! replacement in the field). A load fails on mismatched copies, so that +//! a torn write or corruption on one drive cannot induce the reader to +//! load stale or invalid information from the other drive. This turns the +//! pair of M.2 drives into a single mirrored storage container that fails +//! closed on any disagreement. We accept this because an M.2 failure is +//! considered a non-user-replaceable part failure, for which the solution +//! is an RMA. If this occurs in the field, sush may refuse to run jobs on +//! the sled containing the failed M.2; we report this error to the user, +//! who should replace the sled. //! //! [sush locker]: //! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs From 6ef1b0662dabaaf81c8b953bfe949a290c80062a Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 21:40:28 -0600 Subject: [PATCH 16/38] Update sush: `isolated` & `lonely` are now associated functions Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- sled-agent/src/sush.rs | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69566a16986..9f81aca3fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15663,7 +15663,7 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "sush-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "chrono", "dropshot", @@ -15679,7 +15679,7 @@ dependencies = [ [[package]] name = "sush-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "bytes", "bytesize", @@ -15714,7 +15714,7 @@ dependencies = [ [[package]] name = "sush-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=94762dc96e1ecad8a75f9280461adee701a066d6#94762dc96e1ecad8a75f9280461adee701a066d6" +source = "git+https://github.com/oxidecomputer/sush?rev=2ceb9d92ba2d607018d044ce709cc49b94bf040b#2ceb9d92ba2d607018d044ce709cc49b94bf040b" dependencies = [ "async-trait", "atomicwrites", diff --git a/Cargo.toml b/Cargo.toml index bdd036dd8e8..f52c6434a5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -861,9 +861,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" -sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } -sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6" } -sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "94762dc96e1ecad8a75f9280461adee701a066d6", features = ["embedded"] } +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 75cc0da40a4..6bdbd788452 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -84,7 +84,9 @@ use tokio_util::sync::CancellationToken; use sush_common::targets::{Cubbies, MAX_CUBBY}; use sush_server::executor::PathIsolation; -use sush_server::gossip::{GossipConfig, isolated, lonely, spawn_gossip}; +use sush_server::gossip::{ + GossipConfig, LinkedBaseboards, Universe, spawn_gossip, +}; use sush_server::link::CorpusSource; use sush_server::locker::Locker; use sush_server::output::{JobOutputDir, OutputDirs}; @@ -250,7 +252,7 @@ pub async fn spawn_sush_tasks( // storing. The null locker stores nothing, leaving the // bookmark on the M.2s untouched for the next boot. let seed = seed_gossip(&log, &Locker::null()).await; - (isolated(seed.into_rumors()), lonely()) + (Universe::isolated(seed.into_rumors()), LinkedBaseboards::lonely()) } }; From e50003de55ff32420b0e89264f55690055a3f4a0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 21:49:39 -0600 Subject: [PATCH 17/38] Improve REQUEST_MAX_BODY_BYTES doc Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 6bdbd788452..1f750aaaea8 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -102,8 +102,14 @@ const MGS_POLL_INTERVAL: Duration = Duration::from_secs(30); /// How long to wait for an MGS candidate to answer. const MGS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); -/// Maximum size of a request body the API will accept. The largest thing a -/// client sends is a signed job request or a certificate, both small. +/// Maximum size of a request body the API will accept. The protocol +/// defines no message size limits, so we chose this cap rather than +/// deriving it. The largest bodies a client sends are the command in +/// a signed job request and a PEM-encoded certificate. Neither has +/// exceeded a few KB in practice, so this leaves an order of magnitude +/// of headroom while bounding how much the server needs to buffer for +/// any one request. We should raise it if we start seeing significantly +/// larger requests or certs in the wild. const REQUEST_MAX_BODY_BYTES: usize = 0xFFFF; /// Handles to the Support Shell server's tasks. From 48ada88ead76e5ebc28b8c230ae06f4d5980b8b1 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Tue, 8 Sep 2026 22:00:05 -0600 Subject: [PATCH 18/38] Reject empty roots at config parsing time Co-Authored-By: Claude Mythos 5 --- sled-agent/src/config.rs | 26 +++++++++++++++++++++++--- sled-agent/src/sush.rs | 3 --- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 4b92545cdd1..b69ea5475c0 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -134,7 +134,10 @@ pub struct Config { pub struct SushConfig { /// PEM files holding the trusted root certificates for job requests, /// one certificate per file. A job whose signature does not chain to - /// one of these is refused, so an empty list means no job can ever run. + /// one of these is refused, so an empty list means no job could ever + /// run, and config parsing rejects it. To run a sled with no Support + /// Shell server, omit the `[sush]` section instead. + #[serde(deserialize_with = "SushConfig::nonempty_roots")] pub roots: Vec, /// Where to record job output before an encrypted dataset is available. @@ -169,6 +172,18 @@ impl SushConfig { fn default_max_output_mb() -> u32 { 10 * 1024 } + + fn nonempty_roots<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let roots = Vec::deserialize(deserializer)?; + if roots.is_empty() { + return Err(serde::de::Error::custom( + "sush needs at least one root certificate to run", + )); + } + Ok(roots) + } } #[derive(Debug, thiserror::Error)] @@ -275,16 +290,21 @@ mod test { let sush: SushConfig = toml::from_str( r#" - roots = [] + roots = ["/pkg/sush-root.pem"] ramdisk_dir = "/var/run/sush" ramdisk_max_output_mb = 8 max_output_mb = 128 "#, ) .unwrap(); - assert!(sush.roots.is_empty()); assert_eq!(sush.ramdisk_dir, "/var/run/sush"); assert_eq!(sush.ramdisk_max_output_mb, 8); assert_eq!(sush.max_output_mb, 128); + + // No roots would mean no jobs can ever run, so reject at parse time. + let err = toml::from_str::("roots = []").unwrap_err(); + assert!( + err.to_string().contains("needs at least one root certificate") + ); } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 1f750aaaea8..52fdd845266 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -196,9 +196,6 @@ pub async fn spawn_sush_tasks( ByteCount::from_mebibytes_u32(config.ramdisk_max_output_mb).to_bytes(), )); - if config.roots.is_empty() { - warn!(log, "sush has no root certificates, so no job will ever run"); - } let shutdown = CancellationToken::new(); // A slot that cannot be created is still handed to the locker, which From e544dcd25c1c63064fba25972b9f71b6751a45ec Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 9 Sep 2026 11:36:54 -0600 Subject: [PATCH 19/38] Replace the cubby map each round instead of merging Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 52fdd845266..db17265c0c0 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -368,13 +368,15 @@ async fn promote_output_dir( } } -/// Periodically ask MGS which baseboard sits in each cubby, and -/// publish the map. A job may name its target sled by cubby; -/// this map resolves it. MGS answers at a fixed address within its -/// switch zone's subnet, so each round we ask at that address in every -/// /64 ddmd has learned, and take answers from any that respond. The -/// answers merge into the map, so a round that goes unanswered never -/// erases it. +/// Periodically ask MGS which baseboard sits in each cubby, +/// and publish the map. (A job may name its target sled by cubby; +/// this map resolves it.) MGS answers at a fixed address within its +/// host sled's subnet, so in each round we try that address in every +/// subnet ddmd knows about, and take answers from any that respond. +/// To avoid stale entries (e.g., if a sled moves slots), any non-trivial +/// response overwrites the map, and we merge multiple responses. +/// If no responses are received, we leave the map alone, since +/// we have no newer data to replace it with. async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { let ddm = match DdmClient::localhost(&log) { Ok(ddm) => ddm, @@ -431,7 +433,11 @@ async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { } } } - cubbies.send_modify(|current| current.extend(map)); + // A truthful response is never empty, since the sled + // running this poll must be in a cubby. + if !map.is_empty() { + cubbies.send_replace(map); + } } Err(err) => { warn!( From 898c66101bdf02d6b0d649f2b889edca7776113c Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 9 Sep 2026 11:57:16 -0600 Subject: [PATCH 20/38] Return a result from AvailableDatasetsReceiver::changed Co-Authored-By: Claude Mythos 5 --- sled-agent/config-reconciler/src/handle.rs | 17 +++++------------ sled-agent/src/sush.rs | 9 ++++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index 3c2f71572e2..e97fe45eb76 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -20,7 +20,6 @@ use sled_storage::nested_dataset::NestedDatasetConfig; use sled_storage::nested_dataset::NestedDatasetListOptions; use sled_storage::nested_dataset::NestedDatasetLocation; use slog::Logger; -use slog::warn; use std::collections::HashSet; use std::sync::Arc; use std::sync::OnceLock; @@ -649,19 +648,13 @@ impl AvailableDatasetsReceiver { /// Wait until the set of available datasets may have changed. /// /// Returns immediately if the set has changed since this receiver last - /// observed it. The test variants' datasets never change, so for them this - /// never returns. - pub async fn changed(&mut self, log: &Logger) { + /// observed it, or `Err` if it can never change again because the + /// config reconciler task is gone. The test variants' datasets never + /// change, so for them this never returns. + pub async fn changed(&mut self) -> Result<(), watch::error::RecvError> { match &mut self.inner { AvailableDatasetsReceiverInner::Real(receiver) => { - if receiver.changed().await.is_err() { - warn!( - log, - "config reconciler task is gone; the set of available \ - datasets will never change again", - ); - std::future::pending().await - } + receiver.changed().await } #[cfg(feature = "testing")] AvailableDatasetsReceiverInner::FakeTempDir { .. } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index db17265c0c0..2671fb84f90 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -364,7 +364,14 @@ async fn promote_output_dir( ), } } - available_datasets_rx.changed(&log).await; + if available_datasets_rx.changed().await.is_err() { + warn!( + log, + "no new datasets will appear, so job output will stay \ + on the ramdisk", + ); + return; + } } } From 74815055e191de463c5d83bee9a780d1614061bd Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 9 Sep 2026 13:09:05 -0600 Subject: [PATCH 21/38] TODONE: check default_ramdisk_dir Co-Authored-By: Claude Mythos 5 --- sled-agent/src/config.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index b69ea5475c0..fa53aca70ff 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -158,9 +158,11 @@ pub struct SushConfig { } impl SushConfig { - // TODO: check that this is the right default. `/var/run/oxide` is tmpfs - // and is created during bootstrap, but it exists to hold ZFS key files - // (see `illumos_utils::zfs::KEYPATH_ROOT`), not bulk job output. + /// We must never store job output unencrypted, so default + /// to a directory on the ramdisk. `/var/run` is a tmpfs + /// mounted by the `filesystem/minimal` service, and the + /// `oxide` subdirectory is created during bootstrap for + /// ZFS key files (see `illumos_utils::zfs::KEYPATH_ROOT`). fn default_ramdisk_dir() -> Utf8PathBuf { "/var/run/oxide/sush".into() } From 3179577544ea6b0914d35d3acb91012b67f68bbd Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 08:57:54 -0600 Subject: [PATCH 22/38] Update sush config doc Co-authored-by: Eliza Weisman --- sled-agent/src/config.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index fa53aca70ff..068949b9a9c 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -132,11 +132,12 @@ pub struct Config { #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct SushConfig { - /// PEM files holding the trusted root certificates for job requests, - /// one certificate per file. A job whose signature does not chain to - /// one of these is refused, so an empty list means no job could ever - /// run, and config parsing rejects it. To run a sled with no Support - /// Shell server, omit the `[sush]` section instead. + /// One or more PEM files holding the trusted root certificates for job + /// requests, one certificate per file. Job signatures must have one of + /// these certificates as their root cert to be accepted. + /// + /// This list must not be empty, as at least one root cert is required in + /// order to be able to run sush jobs. #[serde(deserialize_with = "SushConfig::nonempty_roots")] pub roots: Vec, From 37e4597291505a43794ff082ae1573ddb72f842c Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 09:07:17 -0600 Subject: [PATCH 23/38] More declauding Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 2671fb84f90..f63941184a0 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -33,9 +33,9 @@ //! picks whose seed wins, and the others bootstrap from it, each //! taking a slice of the winner's identity space. A restarted sled //! therefore rejoins the rack's universe and replays what it missed -//! without re-executing it. Each sled stores its identity in a -//! _bookmark_, so that a restart resumes it rather than growing a new -//! one on every boot. +//! without re-executing previously run jobs. Each sled stores its identity in +//! a record called a _bookmark_, which is persisted to disk in order to survive +//! the sled restarting. //! //! The bookmark, like every record sush must trust across reboots, lives //! in the [sush locker]. When we write a record to the locker, we actually From 05b110c3e80a91fcfc873e6523672ac380871416 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 09:09:50 -0600 Subject: [PATCH 24/38] Declaude poll_mgs_for_cubbies doc Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index f63941184a0..3507d4a3a08 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -375,15 +375,22 @@ async fn promote_output_dir( } } -/// Periodically ask MGS which baseboard sits in each cubby, -/// and publish the map. (A job may name its target sled by cubby; -/// this map resolves it.) MGS answers at a fixed address within its -/// host sled's subnet, so in each round we try that address in every -/// subnet ddmd knows about, and take answers from any that respond. -/// To avoid stale entries (e.g., if a sled moves slots), any non-trivial -/// response overwrites the map, and we merge multiple responses. -/// If no responses are received, we leave the map alone, since -/// we have no newer data to replace it with. +/// Discovers the baseboard identity in each cubby in the rack and publishes the +/// map over the provided watch channel. +/// +/// This map is used to resolve the sled identity when a job specifies its +/// target sled cubby number. +/// +/// This function may start while one or more MGS services are not available, +/// and must handle failures of both the MGS service or the entire scrimlet +/// gracefully. Therefore, on every poll, this function will attempt to use the +/// fixed MGS address on every subnet curretly known to `ddmd`, and accepts any +/// responses it receives. If a response that contains at least one sled is +/// received, the previously-discovered map is overwritten to avoid leaving +/// behind stale entries for sleds that are no longer present. Multiple +/// responses received within the same poll are merged to produce a single map. +/// If we cannot contact any MGS instance during a poll, the map does not change +/// until a subsequent poll receives a response. async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { let ddm = match DdmClient::localhost(&log) { Ok(ddm) => ddm, From a2b8a97f10e25142fcca5c6dcdb83b899b5ff511 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 09:10:20 -0600 Subject: [PATCH 25/38] Declaude comment Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 3507d4a3a08..983ad01727c 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -447,8 +447,9 @@ async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { } } } - // A truthful response is never empty, since the sled - // running this poll must be in a cubby. + // If this code is running, there must be at least one sled in + // the rack, so we reject any response that does not have at + // least one sled. if !map.is_empty() { cubbies.send_replace(map); } From b0eef7f43d689d7ef4be67849920e4b44b6f8976 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 09:16:32 -0600 Subject: [PATCH 26/38] Declaude comment Co-authored-by: Eliza Weisman --- sled-agent/src/sush.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 983ad01727c..5b03aac64c4 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -138,8 +138,9 @@ impl SushHandles { .config(ConfigDropshot { bind_address, default_request_body_max_bytes: REQUEST_MAX_BODY_BYTES, - // An interactive job holds a websocket open for as long as it runs, - // so a handler must outlive the request that created it. + // While an interactive job is running, the corresponding websocket + // connection must remain open, so HTTP request handlers must + // outlive the requests that created them. default_handler_task_mode: HandlerTaskMode::Detached, log_headers: vec![], compression: Default::default(), From f65febb3a2d56faf3ea27bd7fa7683e9dcea3fa0 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 09:42:39 -0600 Subject: [PATCH 27/38] Clarify universe dominance rule Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 5b03aac64c4..4b39a353bc2 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -29,9 +29,9 @@ //! related messages is called a _universe_: peers in the same universe //! can gossip and converge on its contents; peers in different //! universes cannot gossip at all. Every identity in a universe -//! descends from a single seed. When sleds meet, a deterministic rule -//! picks whose seed wins, and the others bootstrap from it, each -//! taking a slice of the winner's identity space. A restarted sled +//! descends from a single seed. On the first gossip sync, a deterministic +//! rule decides whose universe wins; the loser joins the winner's +//! universe and adopts a slice of its identity space. A restarted sled //! therefore rejoins the rack's universe and replays what it missed //! without re-executing previously run jobs. Each sled stores its identity in //! a record called a _bookmark_, which is persisted to disk in order to survive From 9e058e753a59c088ca51627332b8a56a98290e2e Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 10:13:13 -0600 Subject: [PATCH 28/38] Point to RFD 620 for locker docs Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 4b39a353bc2..c3604ba6466 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -38,22 +38,15 @@ //! the sled restarting. //! //! The bookmark, like every record sush must trust across reboots, lives -//! in the [sush locker]. When we write a record to the locker, we actually -//! write two copies, one to each M.2. When we load a record, it only -//! succeeds if both copies are exact matches, or if one of them is -//! entirely missing (the latter to handle the case of M.2 hardware -//! replacement in the field). A load fails on mismatched copies, so that -//! a torn write or corruption on one drive cannot induce the reader to -//! load stale or invalid information from the other drive. This turns the -//! pair of M.2 drives into a single mirrored storage container that fails -//! closed on any disagreement. We accept this because an M.2 failure is -//! considered a non-user-replaceable part failure, for which the solution -//! is an RMA. If this occurs in the field, sush may refuse to run jobs on -//! the sled containing the failed M.2; we report this error to the user, -//! who should replace the sled. +//! in the [sush locker], described in the [storage section of RFD 620]. +//! This is similar to the [`omicron_ledger::Ledger`], which also holds +//! records across both M.2s, but handles disagreements between the two +//! drives differently. //! //! [sush locker]: //! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs +//! [storage section of RFD 620]: +//! https://rfd.shared.oxide.computer/rfd/0620#_storage use crate::config::SushConfig; use anyhow::Context; From 26409542e6bac3cc7f156618d0435ebd3030a277 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 13:08:05 -0600 Subject: [PATCH 29/38] Clarify encrypted output dir upgrade --- sled-agent/src/sush.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index c3604ba6466..8e54d609d67 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -173,9 +173,9 @@ pub async fn spawn_sush_tasks( ) -> Option { let log = log.new(o!("component" => "sush")); - // Job output starts on the ramdisk, because an encrypted dataset cannot be - // mounted until trust quorum is established, and sush must be useful - // before then. + // Job output starts on the ramdisk, because an encrypted dataset cannot + // be mounted until trust quorum is established, and sush must be useful + // before then; see `promote_output_dir`. if let Err(err) = create_dir_all(&config.ramdisk_dir).await { error!( log, @@ -325,7 +325,8 @@ pub async fn spawn_sush_tasks( /// Start recording new job output on an encrypted debug dataset as soon /// as one is mounted, with a raised size limit. Output already recorded -/// on the ramdisk stays there, readable until reboot. +/// on the ramdisk currently stays there, but should be migrated to the +/// encrypted storage; see sush#69. async fn promote_output_dir( log: Logger, mut available_datasets_rx: AvailableDatasetsReceiver, From 32741d66daad0b42aa0d4aea5d4b7cccb29da36b Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 13:11:50 -0600 Subject: [PATCH 30/38] fmt Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 8e54d609d67..6b027889374 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -132,7 +132,7 @@ impl SushHandles { bind_address, default_request_body_max_bytes: REQUEST_MAX_BODY_BYTES, // While an interactive job is running, the corresponding websocket - // connection must remain open, so HTTP request handlers must + // connection must remain open, so HTTP request handlers must // outlive the requests that created them. default_handler_task_mode: HandlerTaskMode::Detached, log_headers: vec![], @@ -373,8 +373,8 @@ async fn promote_output_dir( /// Discovers the baseboard identity in each cubby in the rack and publishes the /// map over the provided watch channel. /// -/// This map is used to resolve the sled identity when a job specifies its -/// target sled cubby number. +/// This map is used to resolve the sled identity when a job specifies its +/// target sled cubby number. /// /// This function may start while one or more MGS services are not available, /// and must handle failures of both the MGS service or the entire scrimlet From f62cd428b44f5ff771fa96e2ce6256d8fa1af867 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 13:47:04 -0600 Subject: [PATCH 31/38] Don't panic! Failure to build an HTTP client would be weird, but shouldn't take down sled-agent. Log an error and bail instead. Co-Authored-By: Claude Mythos 5 --- sled-agent/src/sush.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 6b027889374..eb6cf073ba6 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -379,7 +379,7 @@ async fn promote_output_dir( /// This function may start while one or more MGS services are not available, /// and must handle failures of both the MGS service or the entire scrimlet /// gracefully. Therefore, on every poll, this function will attempt to use the -/// fixed MGS address on every subnet curretly known to `ddmd`, and accepts any +/// fixed MGS address on every subnet currently known to `ddmd`, and accepts any /// responses it receives. If a response that contains at least one sled is /// received, the previously-discovered map is overwritten to avoid leaving /// behind stale entries for sleds that are no longer present. Multiple @@ -387,21 +387,24 @@ async fn promote_output_dir( /// If we cannot contact any MGS instance during a poll, the map does not change /// until a subsequent poll receives a response. async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { - let ddm = match DdmClient::localhost(&log) { - Ok(ddm) => ddm, - Err(err) => { + let clients = || -> anyhow::Result<(DdmClient, reqwest::Client)> { + let ddm = DdmClient::localhost(&log)?; + let client = reqwest::ClientBuilder::new() + .connect_timeout(MGS_PROBE_TIMEOUT) + .timeout(MGS_PROBE_TIMEOUT) + .build()?; + Ok((ddm, client)) + }; + let (ddm, client) = match clients() { + Ok(clients) => clients, + Err(error) => { error!( log, "not polling MGS, cubby-targeted jobs will not run here"; - "error" => InlineErrorChain::new(&err), + "error" => #%error, ); return; } }; - let client = reqwest::ClientBuilder::new() - .connect_timeout(MGS_PROBE_TIMEOUT) - .timeout(MGS_PROBE_TIMEOUT) - .build() - .expect("failed to build an HTTP client"); loop { match ddm.derive_underlay_subnets_from_prefixes().await { Ok(subnets) => { From e9ff322be00e8d0b9d3f9b38be4587e8fe343e7c Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 13:58:56 -0600 Subject: [PATCH 32/38] TODO: cleanup poll_ddm_for_peer_updates Co-authored-by: Eliza Weisman --- sled-agent/src/bootstrap/bootstore_setup.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs index e833dcf40b2..0c4319f9852 100644 --- a/sled-agent/src/bootstrap/bootstore_setup.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -78,8 +78,13 @@ fn bootstore_network_config_paths( } /// Poll ddmd for the set of peer sleds on the bootstrap network, and -/// publish it to the three consumers that each need it at their own -/// port: the bootstore, the trust quorum node, and sush gossip. +/// publish the discovered peer IP addresses to the bootstore, the trust quorum +/// node, and sush gossip. +// TODO: currently, this function takes three separate watch channels that +// publish a set of socket addresses composed of the same set of IPs, but with +// different port numbers for each consumer. In the future, it might be worth +// refactoring this to just use one `watch` channel to publish the set of IPs +// and have the consumers construct socket addresses themselves. pub async fn poll_ddmd_for_peer_updates( log: Logger, bootstore_node_handle: bootstore::NodeHandle, From 5096283fdb0a2aac773839a779a93103e2385a57 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 10 Sep 2026 17:18:55 -0600 Subject: [PATCH 33/38] Bump sush pin Locker construction now fails if another process is using it. Co-Authored-By: Claude Mythos 5 --- Cargo.lock | 401 +++++++++++++++++++++++++++++++++++--- Cargo.toml | 6 +- sled-agent/src/sush.rs | 11 +- workspace-hack/Cargo.toml | 46 +++-- 4 files changed, 412 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d84a6efad42..039f51167e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -656,6 +656,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "before" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=90e4509dc66330affb38afa74813a8dd500edd93#90e4509dc66330affb38afa74813a8dd500edd93" +dependencies = [ + "bytes", + "dashu-int", + "dsi-bitstream", + "serde", + "serde_json", + "static_assertions", + "suanpan", + "thiserror 2.0.18", +] + [[package]] name = "bhyve_api" version = "0.0.0" @@ -1024,6 +1039,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + [[package]] name = "bzip2" version = "0.4.4" @@ -1703,7 +1724,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2571,6 +2592,25 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dashu-base" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64b04cdfc4c8533100fe00304eb9687173bda47b1f1dac8af12ba13712ed49d" + +[[package]] +name = "dashu-int" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ee98721d5d223e5b64b642dd9588b79d9ef415554b13720308b77d628c3be6" +dependencies = [ + "cfg-if", + "dashu-base", + "num-modular", + "rustversion", + "static_assertions", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -3440,6 +3480,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dsi-bitstream" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f3e8a7ab0f1148cf7a792807d330f1e417dda3f4f122526e609a83925b4823" +dependencies = [ + "num-primitive", +] + [[package]] name = "dsl_auto_type" version = "0.2.0" @@ -3797,7 +3846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4762,6 +4811,8 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -5109,6 +5160,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -5373,7 +5430,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-layer", @@ -6070,7 +6127,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6810,6 +6867,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d317b4b9eb398e6acce275758ec6125535505e7a146fb1a9b8bda2451b0ff4c" +dependencies = [ + "hashbrown 0.17.0", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -8558,7 +8624,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8647,6 +8713,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-modular" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8e500409e6cd603b03e477c26a6caecdc27ac58979a53e881c75eafc079f44" + +[[package]] +name = "num-primitive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0178502a58a2514f927965f80216c0ea2533b5364061544ea8e09b79343d4183" + [[package]] name = "num-rational" version = "0.4.2" @@ -9794,6 +9872,8 @@ dependencies = [ "fmd-adm", "fmd-adm-sys", "futures", + "gateway-client", + "gateway-types", "glob", "guppy", "hex", @@ -9877,6 +9957,9 @@ dependencies = [ "sprockets-tls", "strum 0.27.2", "subprocess", + "sush-api", + "sush-common", + "sush-server", "tar", "test-strategy", "thiserror 2.0.18", @@ -9987,6 +10070,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "crossterm 0.28.1", + "crypto-bigint 0.5.5", "crypto-common 0.1.7", "crypto-common 0.2.2", "ctutils", @@ -9996,6 +10080,7 @@ dependencies = [ "der 0.7.10", "derive_more 2.1.1", "derive_more-impl", + "dice-verifier 0.3.0-pre0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", "digest 0.10.7", "digest 0.11.3", "dof 0.3.0", @@ -10038,10 +10123,12 @@ dependencies = [ "ipnet", "ipnetwork", "itertools 0.13.0", + "itertools 0.15.0", "jiff", "lalrpop-util", "lazy_static", "libc", + "linux-raw-sys 0.12.1", "linux-raw-sys 0.4.15", "log", "managed", @@ -10099,6 +10186,7 @@ dependencies = [ "smallvec 1.15.1", "spin", "spki 0.7.3", + "sprockets-tls", "string_cache", "strum 0.26.3", "strum 0.27.2", @@ -10878,6 +10966,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + [[package]] name = "p256" version = "0.14.0" @@ -10917,6 +11017,20 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct 0.2.0", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "rand_core 0.6.4", + "sha2 0.10.9", +] + [[package]] name = "p521" version = "0.14.0" @@ -11303,7 +11417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cd31dcfdbbd7431a807ef4df6edd6473228e94d5c805e8cf671227a21bad068" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.15.0", "proc-macro2", "quote", "rand 0.8.6", @@ -12187,6 +12301,16 @@ dependencies = [ "psl-types", ] +[[package]] +name = "pwd" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c71c0c79b9701efe4e1e4b563b2016dd4ee789eb99badcb09d61ac4b92e4a2" +dependencies = [ + "libc", + "thiserror 1.0.69", +] + [[package]] name = "qorb" version = "0.4.1" @@ -12237,7 +12361,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -12276,9 +12400,9 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -12491,7 +12615,7 @@ dependencies = [ "indoc 2.0.7", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum 0.26.3", "unicode-segmentation", @@ -12972,6 +13096,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "ron" version = "0.8.1" @@ -13076,6 +13209,30 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rumors" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=90e4509dc66330affb38afa74813a8dd500edd93#90e4509dc66330affb38afa74813a8dd500edd93" +dependencies = [ + "async-stream", + "before", + "bytes", + "ciborium", + "futures", + "futures-util", + "hex", + "itertools 0.14.0", + "rand 0.8.6", + "seq-macro", + "serde", + "sha3 0.12.0", + "smallvec 1.15.1", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tokio-stream", +] + [[package]] name = "russh" version = "0.62.4" @@ -13116,9 +13273,9 @@ dependencies = [ "ml-kem", "module-lattice", "num-bigint", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "pageant", "pbkdf2", "pkcs1 0.8.0-rc.4", @@ -13138,8 +13295,8 @@ dependencies = [ "sha3 0.12.0", "signature 3.0.0", "spki 0.8.0", - "ssh-encoding", - "ssh-key", + "ssh-encoding 0.3.0", + "ssh-key 0.7.0-rc.11", "subtle", "thiserror 2.0.18", "tokio", @@ -13156,7 +13313,7 @@ checksum = "3aec6cb630dbe85d72ffd7bcd95f07e1bd69f9f270ee8adfa1afe443a6331438" dependencies = [ "log", "nix 0.31.2", - "ssh-encoding", + "ssh-encoding 0.3.0", "windows-sys 0.61.2", ] @@ -13236,7 +13393,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13303,7 +13460,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -15022,7 +15179,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -15120,7 +15277,7 @@ checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" [[package]] name = "sprockets-tls" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "anyhow", "attest-data 0.5.0 (git+https://github.com/oxidecomputer/dice-util?rev=4a39ef08d81e5177edee0bddb1146032aa21074d)", @@ -15153,7 +15310,7 @@ dependencies = [ [[package]] name = "sprockets-tls-test-utils" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sprockets.git?rev=a233079e04d9688454486c452b694587a28c5257#a233079e04d9688454486c452b694587a28c5257" +source = "git+https://github.com/oxidecomputer/sprockets.git?rev=98615b86d51f74ce220198eb16c57932fd24e7cb#98615b86d51f74ce220198eb16c57932fd24e7cb" dependencies = [ "camino", "pki-playground 0.2.0 (git+https://github.com/oxidecomputer/pki-playground?rev=7600756029ce046a02c6234aa84ce230cc5eaa04)", @@ -15191,6 +15348,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ssh-cipher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" +dependencies = [ + "cipher 0.4.4", + "ssh-encoding 0.2.0", +] + [[package]] name = "ssh-cipher" version = "0.3.0" @@ -15205,10 +15372,21 @@ dependencies = [ "ctutils", "des", "poly1305 0.9.0", - "ssh-encoding", + "ssh-encoding 0.3.0", "zeroize", ] +[[package]] +name = "ssh-encoding" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" +dependencies = [ + "base64ct", + "pem-rfc7468 0.7.0", + "sha2 0.10.9", +] + [[package]] name = "ssh-encoding" version = "0.3.0" @@ -15224,6 +15402,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ssh-key" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" +dependencies = [ + "ed25519-dalek 2.2.0", + "p256 0.13.2", + "p384 0.13.1", + "p521 0.13.3", + "rand_core 0.6.4", + "rsa 0.9.10", + "sec1 0.7.3", + "serde", + "sha2 0.10.9", + "signature 2.2.0", + "ssh-cipher 0.2.0", + "ssh-encoding 0.2.0", + "subtle", + "zeroize", +] + [[package]] name = "ssh-key" version = "0.7.0-rc.11" @@ -15236,17 +15436,17 @@ dependencies = [ "ed25519-dalek 3.0.0", "hex", "hmac 0.13.0", - "p256", + "p256 0.14.0", "p384 0.14.0", - "p521", + "p521 0.14.0", "rand_core 0.10.0", "rsa 0.10.0-rc.18", "sec1 0.8.1", "sha1 0.11.0", "sha2 0.11.0", "signature 3.0.0", - "ssh-cipher", - "ssh-encoding", + "ssh-cipher 0.3.0", + "ssh-encoding 0.3.0", "zeroize", ] @@ -15427,6 +15627,14 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "suanpan" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rumors?rev=90e4509dc66330affb38afa74813a8dd500edd93#90e4509dc66330affb38afa74813a8dd500edd93" +dependencies = [ + "dashu-int", +] + [[package]] name = "subprocess" version = "0.2.15" @@ -15546,6 +15754,110 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" +[[package]] +name = "sush-api" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=4254c920049959270864c5c97ef4b23d79ea46e1#4254c920049959270864c5c97ef4b23d79ea46e1" +dependencies = [ + "chrono", + "dropshot", + "http-range-header", + "hyper", + "schemars 0.8.22", + "serde", + "sled-hardware-types", + "sush-common", + "thiserror 1.0.69", +] + +[[package]] +name = "sush-common" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=4254c920049959270864c5c97ef4b23d79ea46e1#4254c920049959270864c5c97ef4b23d79ea46e1" +dependencies = [ + "bytes", + "bytesize", + "chrono", + "crypto-bigint 0.5.5", + "ed25519-dalek 2.2.0", + "hex", + "http", + "http-range-header", + "p256 0.13.2", + "pem-rfc7468 0.7.0", + "rand 0.8.6", + "rand_core 0.6.4", + "rlimit", + "rustix 1.1.4", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sha3 0.12.0", + "signature 2.2.0", + "sled-hardware-types", + "slog", + "ssh-key 0.6.7", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite 0.28.0", + "x509-cert", +] + +[[package]] +name = "sush-server" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/sush?rev=4254c920049959270864c5c97ef4b23d79ea46e1#4254c920049959270864c5c97ef4b23d79ea46e1" +dependencies = [ + "async-trait", + "atomicwrites", + "bytes", + "bytesize", + "camino", + "chrono", + "ciborium", + "clap", + "dropshot", + "ed25519-dalek 2.2.0", + "futures", + "http", + "http-body-util", + "http-range-header", + "hyper", + "hyper-util", + "libc", + "lru 0.18.3", + "memmap2", + "p256 0.13.2", + "percent-encoding", + "pwd", + "qorb", + "rand_core 0.6.4", + "rumors", + "rustix 1.1.4", + "rustls", + "schemars 0.8.22", + "serde", + "serde_json", + "sha2 0.10.9", + "sled-hardware-types", + "slog", + "slog-term", + "sprockets-tls", + "ssh-key 0.6.7", + "sush-api", + "sush-common", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-tungstenite 0.28.0", + "tokio-util", + "x509-cert", +] + [[package]] name = "swrite" version = "0.1.0" @@ -15737,7 +16049,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -15757,7 +16069,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -16002,9 +16314,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -16231,6 +16543,18 @@ dependencies = [ "tungstenite 0.23.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.28.0", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -17015,6 +17339,23 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1 0.10.6", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "twox-hash" version = "2.1.2" @@ -18089,7 +18430,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb5a9d26ac8..df454f0a6bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -863,9 +863,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" -sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } -sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b" } -sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "2ceb9d92ba2d607018d044ce709cc49b94bf040b", features = ["embedded"] } +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "4254c920049959270864c5c97ef4b23d79ea46e1" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "4254c920049959270864c5c97ef4b23d79ea46e1" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "4254c920049959270864c5c97ef4b23d79ea46e1", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index eb6cf073ba6..df80e92d23a 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -205,7 +205,16 @@ pub async fn spawn_sush_tasks( ); } } - let locker = Locker::new(&log, slots); + let locker = match Locker::new(&log, slots) { + Ok(locker) => locker, + Err(err) => { + error!( + log, "locker is locked, not starting sush server"; + "error" => InlineErrorChain::new(&err), + ); + return None; + } + }; let GossipInputs { sprockets, measurements, bootstrap_ip, peers } = gossip; diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index fddff9c6066..29aebc94409 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -40,6 +40,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -48,6 +49,7 @@ daft = { version = "0.1.8", features = ["derive", "newtype-uuid1", "oxnet01", "u data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -70,7 +72,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } hex = { version = "0.4.3", features = ["serde"] } hickory-proto = { version = "0.25.2", features = ["serde", "text-parsing"] } @@ -82,7 +84,8 @@ idna = { version = "1.1.0" } indexmap = { version = "2.14.0", features = ["serde"] } ipnet = { version = "2.12.0", features = ["serde"] } ipnetwork = { version = "0.21.1", features = ["schemars", "serde"] } -itertools = { version = "0.13.0" } +itertools-3575ec1268b04181 = { package = "itertools", version = "0.15.0" } +itertools-594e8ee84c453af0 = { package = "itertools", version = "0.13.0" } jiff = { version = "0.2.34", features = ["serde"] } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } @@ -120,7 +123,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -135,9 +138,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -193,6 +197,7 @@ const-oid = { version = "0.9.6", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.18" } crossbeam-utils = { version = "0.8.21" } crossterm = { version = "0.28.1", features = ["event-stream", "serde"] } +crypto-bigint = { version = "0.5.5", features = ["generic-array", "zeroize"] } crypto-common-6f8ce4dd05d13bba = { package = "crypto-common", version = "0.2.2", default-features = false, features = ["getrandom"] } crypto-common-c65f7effa3be6d31 = { package = "crypto-common", version = "0.1.7", default-features = false, features = ["getrandom", "std"] } ctutils = { version = "0.4.2", default-features = false, features = ["subtle"] } @@ -202,6 +207,7 @@ data-encoding = { version = "2.10.0" } der = { version = "0.7.10", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } derive_more = { version = "2.1.1", features = ["as_ref", "is_variant"] } derive_more-impl = { version = "2.1.1", features = ["as_ref", "is_variant"] } +dice-verifier = { git = "https://github.com/oxidecomputer/dice-util", rev = "4a39ef08d81e5177edee0bddb1146032aa21074d", default-features = false, features = ["ipcc", "mock"] } digest-93f6ce9d446188ac = { package = "digest", version = "0.10.7", features = ["mac", "oid", "std"] } digest-a6292c17cd707f01 = { package = "digest", version = "0.11.3", features = ["alloc", "mac", "oid", "rand_core"] } ed25519-dalek = { version = "2.2.0", features = ["digest", "pem", "rand_core"] } @@ -224,7 +230,7 @@ getrandom-6f8ce4dd05d13bba = { package = "getrandom", version = "0.2.17", defaul getrandom-9fbad63c4bcf4a8f = { package = "getrandom", version = "0.4.3", default-features = false, features = ["std", "sys_rng", "wasm_js"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15.5" } -hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0", default-features = false, features = ["default-hasher"] } +hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17.0" } hashbrown-986da7b5efc2b80e = { package = "hashbrown", version = "0.16.1" } heck = { version = "0.4.1", features = ["unicode"] } hex = { version = "0.4.3", features = ["serde"] } @@ -237,7 +243,8 @@ idna = { version = "1.1.0" } indexmap = { version = "2.14.0", features = ["serde"] } ipnet = { version = "2.12.0", features = ["serde"] } ipnetwork = { version = "0.21.1", features = ["schemars", "serde"] } -itertools = { version = "0.13.0" } +itertools-3575ec1268b04181 = { package = "itertools", version = "0.15.0" } +itertools-594e8ee84c453af0 = { package = "itertools", version = "0.13.0" } jiff = { version = "0.2.34", features = ["serde"] } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } @@ -275,7 +282,7 @@ regex-syntax = { version = "0.8.10" } reqwest-594e8ee84c453af0 = { package = "reqwest", version = "0.13.2", features = ["blocking", "cookies", "json", "query", "stream"] } reqwest-5ef9efb8ec2df382 = { package = "reqwest", version = "0.12.28", features = ["blocking", "json", "stream"] } rsa = { version = "0.9.10", features = ["serde", "sha2"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["fs"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", features = ["process", "pty", "termios"] } rustls = { version = "0.23.41" } schemars = { version = "0.8.22", features = ["bytes", "chrono", "semver", "url", "uuid1"] } scopeguard = { version = "1.2.0" } @@ -291,9 +298,10 @@ simd-adler32 = { version = "0.3.8", default-features = false, features = ["std"] similar = { version = "2.7.0", features = ["bytes", "inline", "unicode"] } slab = { version = "0.4.12" } slog = { version = "2.8.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } -smallvec = { version = "1.15.1", default-features = false, features = ["const_new"] } +smallvec = { version = "1.15.1", default-features = false, features = ["const_new", "union"] } spin = { version = "0.9.8" } spki = { version = "0.7.3", default-features = false, features = ["pem", "std"] } +sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "98615b86d51f74ce220198eb16c57932fd24e7cb", default-features = false, features = ["ipcc"] } string_cache = { version = "0.8.9" } strum-2f80eeee3b1b6c7e = { package = "strum", version = "0.26.3", features = ["derive"] } strum-754bda37e0fb3874 = { package = "strum", version = "0.27.2", features = ["derive"] } @@ -335,13 +343,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-linux-gnu.build-dependencies] @@ -350,13 +359,14 @@ dof-468e82937335b1c9 = { package = "dof", version = "0.3.0", default-features = dof-9fbad63c4bcf4a8f = { package = "dof", version = "0.4.0", default-features = false, features = ["des"] } hyper-rustls = { version = "0.27.7", features = ["http2"] } hyper-util = { version = "0.1.20", features = ["full"] } -linux-raw-sys = { version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } +linux-raw-sys-5ef9efb8ec2df382 = { package = "linux-raw-sys", version = "0.12.1", default-features = false, features = ["auxvec", "elf", "errno", "general", "ioctl", "no_std", "prctl"] } +linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4.15", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "system"] } miniz_oxide = { version = "0.8.9", default-features = false, features = ["simd", "with-alloc"] } mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.dependencies] @@ -369,7 +379,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-apple-darwin.build-dependencies] @@ -382,7 +392,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.dependencies] @@ -395,7 +405,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.aarch64-apple-darwin.build-dependencies] @@ -408,7 +418,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } [target.x86_64-unknown-illumos.dependencies] @@ -426,7 +436,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } @@ -446,7 +456,7 @@ mio = { version = "1.2.0", features = ["net", "os-ext"] } nix = { version = "0.31.2", default-features = false, features = ["mman"] } object = { version = "0.37.3", default-features = false, features = ["read", "std"] } rustix-d585fab2519d2d1 = { package = "rustix", version = "0.38.44", features = ["fs", "stdio", "system", "termios"] } -rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio", "termios"] } +rustix-dff4ba8e3ae991db = { package = "rustix", version = "1.1.4", default-features = false, features = ["stdio"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["aws-lc-rs"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } winnow = { version = "1.0.3" } From e014b50fc86ab62b78e74da0d357e88e048048f3 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Wed, 4 Nov 2026 15:33:42 -0700 Subject: [PATCH 34/38] Choose sush trust anchors by platform identity Replace the sush config's `roots` with production and staging trust anchors. Sleds whose platform identity chains to the staging root also trust the staging anchors; every other sled trusts only the production ones. Co-authored-by: Claude --- sled-agent/src/config.rs | 165 ++++++++++++------ sled-agent/src/sush.rs | 115 +++++++++++- smf/sled-agent/gimlet-standalone/config.toml | 7 +- .../gimlet-standalone/sush-sandbox-s1.pem | 10 ++ ...sandbox.pem => sush-sandbox-untrusted.pem} | 0 smf/sled-agent/gimlet/config.toml | 7 +- smf/sled-agent/gimlet/sush-sandbox-s1.pem | 10 ++ ...sandbox.pem => sush-sandbox-untrusted.pem} | 0 smf/sled-agent/non-gimlet/config.toml | 7 +- smf/sled-agent/non-gimlet/sush-sandbox-s1.pem | 10 ++ ...sandbox.pem => sush-sandbox-untrusted.pem} | 0 11 files changed, 273 insertions(+), 58 deletions(-) create mode 100644 smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem rename smf/sled-agent/gimlet-standalone/{sush-sandbox.pem => sush-sandbox-untrusted.pem} (100%) create mode 100644 smf/sled-agent/gimlet/sush-sandbox-s1.pem rename smf/sled-agent/gimlet/{sush-sandbox.pem => sush-sandbox-untrusted.pem} (100%) create mode 100644 smf/sled-agent/non-gimlet/sush-sandbox-s1.pem rename smf/sled-agent/non-gimlet/{sush-sandbox.pem => sush-sandbox-untrusted.pem} (100%) diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 6ee3f008713..483d62ee114 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -349,14 +349,22 @@ pub struct Config { #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct SushConfig { - /// One or more PEM files holding the trusted root certificates for job - /// requests, one certificate per file. Job signatures must have one of - /// these certificates as their root cert to be accepted. - /// - /// This list must not be empty, as at least one root cert is required in - /// order to be able to run sush jobs. - #[serde(deserialize_with = "SushConfig::nonempty_roots")] - pub roots: Vec, + /// A PEM file containing the production platform identity root + /// certificate. + pub production_platform_root: Utf8PathBuf, + + /// A PEM file containing the staging platform identity root + /// certificate. + pub staging_platform_root: Utf8PathBuf, + + /// PEM files holding the certificates that *every* sled trusts + /// to sign jobs, one certificate per file, in root-to-leaf order. + pub production_trust_anchors: Vec, + + /// PEM files holding the certificates that *only staging-signed* sleds + /// (i.e., Oxide lab systems) trust to sign jobs, in root-to-leaf order. + #[serde(default)] + pub staging_trust_anchors: Vec, /// Where to record job output before an encrypted dataset is available. /// Must be on a ramdisk so that it is never persisted unencrypted. @@ -392,18 +400,6 @@ impl SushConfig { fn default_max_output_mb() -> u32 { 10 * 1024 } - - fn nonempty_roots<'de, D: serde::Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - let roots = Vec::deserialize(deserializer)?; - if roots.is_empty() { - return Err(serde::de::Error::custom( - "sush needs at least one root certificate to run", - )); - } - Ok(roots) - } } #[derive(Debug, thiserror::Error)] @@ -569,13 +565,13 @@ mod test { use slog_error_chain::InlineErrorChain; - #[test] - fn test_smf_configs() { + /// Parse every sled-agent config we ship. + fn smf_configs() -> Vec<(Utf8PathBuf, Config)> { let manifest = std::env::var("CARGO_MANIFEST_DIR") .expect("Cannot access manifest directory"); let smf = Utf8PathBuf::from(manifest).join("../smf/sled-agent"); - let mut configs_seen = 0; + let mut configs = Vec::new(); for variant in smf.read_dir_utf8().unwrap() { let variant = variant.unwrap(); if variant.file_type().unwrap().is_dir() { @@ -583,50 +579,117 @@ mod test { let entry = entry.unwrap(); if entry.file_name() == "config.toml" { let path = entry.path(); - Config::from_file(&path).unwrap_or_else(|e| { - panic!( - "Failed to parse config {path}: {}", - InlineErrorChain::new(&e) - ) - }); - configs_seen += 1; + let config = + Config::from_file(&path).unwrap_or_else(|e| { + panic!( + "Failed to parse config {path}: {}", + InlineErrorChain::new(&e) + ) + }); + configs.push((path.to_owned(), config)); } } } } - assert!(configs_seen > 0, "No sled-agent configs found"); + configs + } + + #[test] + fn test_smf_configs() { + assert!(!smf_configs().is_empty(), "No sled-agent configs found"); } #[test] fn test_sush_config() { - // Only `roots` is required. The rest have defaults. - let sush: SushConfig = - toml::from_str(r#"roots = ["/pkg/sush-root.pem"]"#).unwrap(); - assert_eq!(sush.roots, vec![Utf8PathBuf::from("/pkg/sush-root.pem")]); - assert_eq!(sush.ramdisk_dir, SushConfig::default_ramdisk_dir()); + // Only the platform roots and the production trust anchors are + // required. The rest have defaults. + const REQUIRED: &str = r#" + production_platform_root = "/idcerts/production.pem" + staging_platform_root = "/idcerts/staging.pem" + production_trust_anchors = ["/pkg/sush-root.pem"] + "#; + let SushConfig { + production_platform_root, + staging_platform_root, + production_trust_anchors, + staging_trust_anchors, + ramdisk_dir, + ramdisk_max_output_mb, + max_output_mb, + } = toml::from_str(REQUIRED).unwrap(); + assert_eq!(production_platform_root, "/idcerts/production.pem"); + assert_eq!(staging_platform_root, "/idcerts/staging.pem"); assert_eq!( - sush.ramdisk_max_output_mb, + production_trust_anchors, + vec![Utf8PathBuf::from("/pkg/sush-root.pem")] + ); + assert!(staging_trust_anchors.is_empty()); + assert_eq!(ramdisk_dir, SushConfig::default_ramdisk_dir()); + assert_eq!( + ramdisk_max_output_mb, SushConfig::default_ramdisk_max_output_mb() ); - assert_eq!(sush.max_output_mb, SushConfig::default_max_output_mb()); - - let sush: SushConfig = toml::from_str( - r#" - roots = ["/pkg/sush-root.pem"] + assert_eq!(max_output_mb, SushConfig::default_max_output_mb()); + + let SushConfig { + production_platform_root: _, + staging_platform_root: _, + production_trust_anchors: _, + staging_trust_anchors, + ramdisk_dir, + ramdisk_max_output_mb, + max_output_mb, + } = toml::from_str(&format!( + r#"{REQUIRED} + staging_trust_anchors = ["/pkg/sush-staging.pem"] ramdisk_dir = "/var/run/sush" ramdisk_max_output_mb = 8 max_output_mb = 128 "#, - ) + )) .unwrap(); - assert_eq!(sush.ramdisk_dir, "/var/run/sush"); - assert_eq!(sush.ramdisk_max_output_mb, 8); - assert_eq!(sush.max_output_mb, 128); - - // No roots would mean no jobs can ever run, so reject at parse time. - let err = toml::from_str::("roots = []").unwrap_err(); - assert!( - err.to_string().contains("needs at least one root certificate") + assert_eq!( + staging_trust_anchors, + vec![Utf8PathBuf::from("/pkg/sush-staging.pem")] ); + assert_eq!(ramdisk_dir, "/var/run/sush"); + assert_eq!(ramdisk_max_output_mb, 8); + assert_eq!(max_output_mb, 128); + + // No anchors means no jobs, but is allowed. + let SushConfig { + production_platform_root: _, + staging_platform_root: _, + production_trust_anchors, + .. + } = toml::from_str( + r#" + production_platform_root = "/idcerts/production.pem" + staging_platform_root = "/idcerts/staging.pem" + production_trust_anchors = [] + "#, + ) + .unwrap(); + assert!(production_trust_anchors.is_empty()); + + // A staging anchor listed as a production one would let + // staging-signed jobs run on production sleds. + for (path, config) in smf_configs() { + let Some(SushConfig { + production_trust_anchors, + staging_trust_anchors, + .. + }) = config.sush + else { + continue; + }; + for anchor in &staging_trust_anchors { + assert!( + !production_trust_anchors.contains(anchor), + "{path}: {anchor} is both a production and a staging \ + sush trust anchor", + ); + } + } } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 4e9235e7f29..7cb1572fa9d 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -6,7 +6,8 @@ //! //! See RFD 620. This server runs jobs targeted at this sled on behalf of //! Oxide support. A job arrives signed, and this sled runs it only when -//! the signature chains to one of the root certificates in its config. +//! the signature chains to one of the trust anchors in its config; see +//! `trust_anchors` for which ones. //! The server is deliberately independent of the rest of the control plane, //! because it is one of the tools you reach for when the control plane is //! the thing that is broken. It therefore starts during bootstrap, before @@ -49,8 +50,9 @@ //! https://rfd.shared.oxide.computer/rfd/0620#_storage use crate::config::SushConfig; -use anyhow::Context; -use camino::Utf8PathBuf; +use anyhow::{Context, ensure}; +use camino::{Utf8Path, Utf8PathBuf}; +use dice_verifier::verify_cert_chain; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer, ServerBuilder}; use gateway_client::Client as MgsClient; use gateway_types::component::SpType; @@ -64,16 +66,21 @@ use sled_agent_measurements::MeasurementsHandle; use sled_hardware_types::BaseboardId; use slog::{Logger, debug, error, info, o, warn}; use slog_error_chain::InlineErrorChain; +use sprockets_tls::ipcc::Ipcc; use sprockets_tls::keys::SprocketsConfig; use std::collections::BTreeSet; +use std::fs::read; use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use std::time::Duration; use tokio::fs::create_dir_all; use tokio::spawn; use tokio::sync::watch; +use tokio::task::spawn_blocking; use tokio::time::sleep; use tokio_util::sync::CancellationToken; +use x509_cert::Certificate; +use x509_cert::der::{Decode as _, DecodePem as _, Reader as _, SliceReader}; use sush_common::targets::{Cubbies, MAX_CUBBY}; use sush_server::executor::PathIsolation; @@ -262,6 +269,7 @@ pub async fn spawn_sush_tasks( } }; + let anchors = trust_anchors(&log, config).await; let (tx_cubbies, rx_cubbies) = watch::channel(Cubbies::new()); let mut manager = match JobManager::new( log.clone(), @@ -272,7 +280,7 @@ pub async fn spawn_sush_tasks( universe, linked, &locker, - &config.roots, + &anchors, shutdown.clone(), ) .await @@ -471,3 +479,102 @@ async fn poll_mgs_for_cubbies(log: Logger, cubbies: watch::Sender) { sleep(MGS_POLL_INTERVAL).await; } } + +/// Choose the certificates this sled will trust to sign sush jobs. +/// Every sled trusts the production anchors, and sleds whose platform +/// identity (fetched here) chains to the staging root trust the staging +/// anchors as well. This allows, e.g., support to run production-signed +/// jobs on lab systems, but does not allow engineers to run staging-signed +/// jobs on production systems. +async fn trust_anchors(log: &Logger, config: &SushConfig) -> Vec { + let SushConfig { + production_platform_root, + staging_platform_root, + production_trust_anchors, + staging_trust_anchors, + ramdisk_dir: _, + ramdisk_max_output_mb: _, + max_output_mb: _, + } = config.clone(); + let mut anchors = production_trust_anchors; + match spawn_blocking(move || { + is_sled_staging_signed( + &production_platform_root, + &staging_platform_root, + ) + }) + .await + .context("fetching the platform identity") + .and_then(|result| result) + { + Ok(false) => info!( + log, + "production platform identity, trusting only production job signers" + ), + Ok(true) => { + info!( + log, + "staging platform identity, trusting staging job signers too" + ); + anchors.extend(staging_trust_anchors); + } + Err(err) => warn!( + log, "unknown platform identity, trusting only production job signers"; + "error" => InlineErrorChain::new(err.as_ref()), + ), + } + anchors +} + +/// Whether this sled's platform identity, read from the RoT, chains to +/// the `staging` root rather than the `production` one. Uses IPCC, so +/// performs blocking I/O. +fn is_sled_staging_signed( + production: &Utf8Path, + staging: &Utf8Path, +) -> anyhow::Result { + let ipcc = Ipcc::new().context("opening IPCC")?; + let chain_der = + ipcc.rot_get_tq_cert_chain().context("fetching the TQ cert chain")?; + let chain = der_cert_chain(&chain_der)?; + let production_root = read_cert(production)?; + let staging_root = read_cert(staging)?; + ensure!(is_staging_cert(&staging_root), "{staging} is not a staging root"); + ensure!( + !is_staging_cert(&production_root), + "{production} is a staging root, not production" + ); + let roots = [production_root, staging_root.clone()]; + let root = verify_cert_chain(&chain, Some(&roots)) + .context("verifying the platform identity")?; + Ok(*root == staging_root) +} + +/// Whether a platform identity root's subject includes "Staging", e.g., +/// "Platform Identity Staging Root A". (Production roots' subjects do +/// not include "Production"). This is used only to fail closed in case +/// roots are accidentally swapped in the config file; it is not used as +/// a verification criterion. +fn is_staging_cert(root: &Certificate) -> bool { + root.tbs_certificate.subject.to_string().contains("Staging") +} + +/// Read a PEM file holding one certificate. +fn read_cert(path: &Utf8Path) -> anyhow::Result { + let pem = read(path).with_context(|| format!("reading {path}"))?; + Certificate::from_pem(&pem).with_context(|| format!("parsing {path}")) +} + +/// Parse a concatenated series of DER certs, as the RoT returns. +fn der_cert_chain(bytes: &[u8]) -> anyhow::Result> { + let mut chain = Vec::new(); + let mut reader = + SliceReader::new(bytes).context("reading the TQ cert chain")?; + while !reader.is_finished() { + chain.push( + Certificate::decode(&mut reader) + .context("parsing the TQ cert chain")?, + ); + } + Ok(chain) +} diff --git a/smf/sled-agent/gimlet-standalone/config.toml b/smf/sled-agent/gimlet-standalone/config.toml index c8af370d0c6..3ca836ad998 100644 --- a/smf/sled-agent/gimlet-standalone/config.toml +++ b/smf/sled-agent/gimlet-standalone/config.toml @@ -78,4 +78,9 @@ enforce = "Enforced" # Support Shell (RFD 620). TODO: replace the sandbox root with the # production certificate. [sush] -roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] +production_platform_root = "/usr/share/oxide/idcerts/production.pem" +staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" +production_trust_anchors = [ + "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", + "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", +] diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem new file mode 100644 index 00000000000..5205bb84253 --- /dev/null +++ b/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy +NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg +U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl +6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC +tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG +Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem similarity index 100% rename from smf/sled-agent/gimlet-standalone/sush-sandbox.pem rename to smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem diff --git a/smf/sled-agent/gimlet/config.toml b/smf/sled-agent/gimlet/config.toml index 40c0b17322c..c8e3e6aeed4 100644 --- a/smf/sled-agent/gimlet/config.toml +++ b/smf/sled-agent/gimlet/config.toml @@ -74,4 +74,9 @@ enforce = "Enforced" # Support Shell (RFD 620). TODO: replace the sandbox root with the # production certificate. [sush] -roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] +production_platform_root = "/usr/share/oxide/idcerts/production.pem" +staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" +production_trust_anchors = [ + "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", + "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", +] diff --git a/smf/sled-agent/gimlet/sush-sandbox-s1.pem b/smf/sled-agent/gimlet/sush-sandbox-s1.pem new file mode 100644 index 00000000000..5205bb84253 --- /dev/null +++ b/smf/sled-agent/gimlet/sush-sandbox-s1.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy +NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg +U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl +6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC +tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG +Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/sush-sandbox.pem b/smf/sled-agent/gimlet/sush-sandbox-untrusted.pem similarity index 100% rename from smf/sled-agent/gimlet/sush-sandbox.pem rename to smf/sled-agent/gimlet/sush-sandbox-untrusted.pem diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index 5c146bd74c0..f7a49c16191 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -144,4 +144,9 @@ enforce = "Enforced" # Support Shell (RFD 620). TODO: replace the sandbox root with the # production certificate. [sush] -roots = ["/opt/oxide/sled-agent/pkg/sush-sandbox.pem"] +production_platform_root = "/usr/share/oxide/idcerts/production.pem" +staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" +production_trust_anchors = [ + "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", + "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", +] diff --git a/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem b/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem new file mode 100644 index 00000000000..5205bb84253 --- /dev/null +++ b/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw +UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy +NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg +U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl +6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC +tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG +Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== +-----END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/sush-sandbox.pem b/smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem similarity index 100% rename from smf/sled-agent/non-gimlet/sush-sandbox.pem rename to smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem From 19d21a1efb0a77f7f8b880f1a4570383a16fa979 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Thu, 5 Nov 2026 15:50:29 -0700 Subject: [PATCH 35/38] Trust the production and lab job signing keys See https://github.com/oxidecomputer/evidence-room/issues/251 Co-authored-by: Claude --- smf/sled-agent/gimlet-standalone/config.toml | 9 +++------ .../gimlet-standalone/support-shell-lab-a1.pem | 11 +++++++++++ .../gimlet-standalone/support-shell-production-a1.pem | 11 +++++++++++ smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem | 10 ---------- .../gimlet-standalone/sush-sandbox-untrusted.pem | 11 ----------- smf/sled-agent/gimlet/config.toml | 9 +++------ smf/sled-agent/gimlet/support-shell-lab-a1.pem | 11 +++++++++++ smf/sled-agent/gimlet/support-shell-production-a1.pem | 11 +++++++++++ smf/sled-agent/gimlet/sush-sandbox-s1.pem | 10 ---------- smf/sled-agent/gimlet/sush-sandbox-untrusted.pem | 11 ----------- smf/sled-agent/non-gimlet/config.toml | 9 +++------ smf/sled-agent/non-gimlet/support-shell-lab-a1.pem | 11 +++++++++++ .../non-gimlet/support-shell-production-a1.pem | 11 +++++++++++ smf/sled-agent/non-gimlet/sush-sandbox-s1.pem | 10 ---------- smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem | 11 ----------- 15 files changed, 75 insertions(+), 81 deletions(-) create mode 100644 smf/sled-agent/gimlet-standalone/support-shell-lab-a1.pem create mode 100644 smf/sled-agent/gimlet-standalone/support-shell-production-a1.pem delete mode 100644 smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem delete mode 100644 smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem create mode 100644 smf/sled-agent/gimlet/support-shell-lab-a1.pem create mode 100644 smf/sled-agent/gimlet/support-shell-production-a1.pem delete mode 100644 smf/sled-agent/gimlet/sush-sandbox-s1.pem delete mode 100644 smf/sled-agent/gimlet/sush-sandbox-untrusted.pem create mode 100644 smf/sled-agent/non-gimlet/support-shell-lab-a1.pem create mode 100644 smf/sled-agent/non-gimlet/support-shell-production-a1.pem delete mode 100644 smf/sled-agent/non-gimlet/sush-sandbox-s1.pem delete mode 100644 smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem diff --git a/smf/sled-agent/gimlet-standalone/config.toml b/smf/sled-agent/gimlet-standalone/config.toml index 3ca836ad998..ec2373749bc 100644 --- a/smf/sled-agent/gimlet-standalone/config.toml +++ b/smf/sled-agent/gimlet-standalone/config.toml @@ -75,12 +75,9 @@ attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" -# Support Shell (RFD 620). TODO: replace the sandbox root with the -# production certificate. +# Support Shell (RFD 620). [sush] production_platform_root = "/usr/share/oxide/idcerts/production.pem" staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" -production_trust_anchors = [ - "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", - "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", -] +production_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-production-a1.pem"] +staging_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-lab-a1.pem"] diff --git a/smf/sled-agent/gimlet-standalone/support-shell-lab-a1.pem b/smf/sled-agent/gimlet-standalone/support-shell-lab-a1.pem new file mode 100644 index 00000000000..9c6fb099b7f --- /dev/null +++ b/smf/sled-agent/gimlet-standalone/support-shell-lab-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBnTCCAUSgAwIBAgIUYUmMPRqGDZzMRgqeCasZcAciVcMwCgYIKoZIzj0EAwIw +TTELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +HTAbBgNVBAMMFFN1cHBvcnQgU2hlbGwgTGFiIEExMCIYDzIwMjYxMTA1MTYxNzIw +WhgPMjEyNjEwMTIxNjE3MjBaME0xCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZPeGlk +ZSBDb21wdXRlciBDb21wYW55MR0wGwYDVQQDDBRTdXBwb3J0IFNoZWxsIExhYiBB +MTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJBveaz9eWJ6WTVIGzrkuSy6FiyV +jzTlW3J2Nn83C+V2at7w1ZR6SQhqh72iTUHQdFmoMfbVD/g5zvr7I6tgfMgwCgYI +KoZIzj0EAwIDRwAwRAIgXRsjYcYmwgRdrBLvGyslg5sPtJHR7Aan3IkIxSwYaHUC +IB8Jpc37a3S8RFcMHwFOuZUaRLPVjPNztT3rHCmJVQnE +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet-standalone/support-shell-production-a1.pem b/smf/sled-agent/gimlet-standalone/support-shell-production-a1.pem new file mode 100644 index 00000000000..ca3189cdbd0 --- /dev/null +++ b/smf/sled-agent/gimlet-standalone/support-shell-production-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBrDCCAVKgAwIBAgIUabiqiIIFqIh5XY3mMFSJUC8yjicwCgYIKoZIzj0EAwIw +VDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +JDAiBgNVBAMMG1N1cHBvcnQgU2hlbGwgUHJvZHVjdGlvbiBBMTAiGA8yMDI2MTEw +NTE2MjEwM1oYDzIxMjYxMDEyMTYyMTAzWjBUMQswCQYDVQQGEwJVUzEfMB0GA1UE +CgwWT3hpZGUgQ29tcHV0ZXIgQ29tcGFueTEkMCIGA1UEAwwbU3VwcG9ydCBTaGVs +bCBQcm9kdWN0aW9uIEExMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAER0R2J/71 +7Ok2C9o3mspibhnYdOEfYwpV6ttuCOsOg1J4btV/rTiw8mf1RhNVJlvajDVyvYlZ +y9M97jn08z+X4jAKBggqhkjOPQQDAgNIADBFAiAQAKpYlsB7/VE9VHkI/VXSa/mq +iIiQ2DGOAhkmj3T0xgIhAJtZE+9HiCbjWvVYWb9VrIUak52BFq3pLdOso7efEl1d +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem deleted file mode 100644 index 5205bb84253..00000000000 --- a/smf/sled-agent/gimlet-standalone/sush-sandbox-s1.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy -NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg -U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl -6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC -tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG -Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== ------END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem b/smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem deleted file mode 100644 index 17b6a761ef2..00000000000 --- a/smf/sled-agent/gimlet-standalone/sush-sandbox-untrusted.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 -NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP -eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy -b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg -m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ -ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk -t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= ------END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/config.toml b/smf/sled-agent/gimlet/config.toml index c8e3e6aeed4..fb6bc9dda6d 100644 --- a/smf/sled-agent/gimlet/config.toml +++ b/smf/sled-agent/gimlet/config.toml @@ -71,12 +71,9 @@ attest = { which = "ipcc" } roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"] enforce = "Enforced" -# Support Shell (RFD 620). TODO: replace the sandbox root with the -# production certificate. +# Support Shell (RFD 620). [sush] production_platform_root = "/usr/share/oxide/idcerts/production.pem" staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" -production_trust_anchors = [ - "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", - "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", -] +production_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-production-a1.pem"] +staging_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-lab-a1.pem"] diff --git a/smf/sled-agent/gimlet/support-shell-lab-a1.pem b/smf/sled-agent/gimlet/support-shell-lab-a1.pem new file mode 100644 index 00000000000..9c6fb099b7f --- /dev/null +++ b/smf/sled-agent/gimlet/support-shell-lab-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBnTCCAUSgAwIBAgIUYUmMPRqGDZzMRgqeCasZcAciVcMwCgYIKoZIzj0EAwIw +TTELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +HTAbBgNVBAMMFFN1cHBvcnQgU2hlbGwgTGFiIEExMCIYDzIwMjYxMTA1MTYxNzIw +WhgPMjEyNjEwMTIxNjE3MjBaME0xCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZPeGlk +ZSBDb21wdXRlciBDb21wYW55MR0wGwYDVQQDDBRTdXBwb3J0IFNoZWxsIExhYiBB +MTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJBveaz9eWJ6WTVIGzrkuSy6FiyV +jzTlW3J2Nn83C+V2at7w1ZR6SQhqh72iTUHQdFmoMfbVD/g5zvr7I6tgfMgwCgYI +KoZIzj0EAwIDRwAwRAIgXRsjYcYmwgRdrBLvGyslg5sPtJHR7Aan3IkIxSwYaHUC +IB8Jpc37a3S8RFcMHwFOuZUaRLPVjPNztT3rHCmJVQnE +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/support-shell-production-a1.pem b/smf/sled-agent/gimlet/support-shell-production-a1.pem new file mode 100644 index 00000000000..ca3189cdbd0 --- /dev/null +++ b/smf/sled-agent/gimlet/support-shell-production-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBrDCCAVKgAwIBAgIUabiqiIIFqIh5XY3mMFSJUC8yjicwCgYIKoZIzj0EAwIw +VDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +JDAiBgNVBAMMG1N1cHBvcnQgU2hlbGwgUHJvZHVjdGlvbiBBMTAiGA8yMDI2MTEw +NTE2MjEwM1oYDzIxMjYxMDEyMTYyMTAzWjBUMQswCQYDVQQGEwJVUzEfMB0GA1UE +CgwWT3hpZGUgQ29tcHV0ZXIgQ29tcGFueTEkMCIGA1UEAwwbU3VwcG9ydCBTaGVs +bCBQcm9kdWN0aW9uIEExMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAER0R2J/71 +7Ok2C9o3mspibhnYdOEfYwpV6ttuCOsOg1J4btV/rTiw8mf1RhNVJlvajDVyvYlZ +y9M97jn08z+X4jAKBggqhkjOPQQDAgNIADBFAiAQAKpYlsB7/VE9VHkI/VXSa/mq +iIiQ2DGOAhkmj3T0xgIhAJtZE+9HiCbjWvVYWb9VrIUak52BFq3pLdOso7efEl1d +-----END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/sush-sandbox-s1.pem b/smf/sled-agent/gimlet/sush-sandbox-s1.pem deleted file mode 100644 index 5205bb84253..00000000000 --- a/smf/sled-agent/gimlet/sush-sandbox-s1.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy -NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg -U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl -6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC -tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG -Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== ------END CERTIFICATE----- diff --git a/smf/sled-agent/gimlet/sush-sandbox-untrusted.pem b/smf/sled-agent/gimlet/sush-sandbox-untrusted.pem deleted file mode 100644 index 17b6a761ef2..00000000000 --- a/smf/sled-agent/gimlet/sush-sandbox-untrusted.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 -NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP -eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy -b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg -m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ -ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk -t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= ------END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index f7a49c16191..4295d5b2c7a 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -141,12 +141,9 @@ attest = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/test-alias-1.k roots = ["/opt/oxide/sled-agent/pkg/test-root-a.cert.pem"] enforce = "Enforced" -# Support Shell (RFD 620). TODO: replace the sandbox root with the -# production certificate. +# Support Shell (RFD 620). [sush] production_platform_root = "/usr/share/oxide/idcerts/production.pem" staging_platform_root = "/usr/share/oxide/idcerts/staging.pem" -production_trust_anchors = [ - "/opt/oxide/sled-agent/pkg/sush-sandbox-untrusted.pem", - "/opt/oxide/sled-agent/pkg/sush-sandbox-s1.pem", -] +production_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-production-a1.pem"] +staging_trust_anchors = ["/opt/oxide/sled-agent/pkg/support-shell-lab-a1.pem"] diff --git a/smf/sled-agent/non-gimlet/support-shell-lab-a1.pem b/smf/sled-agent/non-gimlet/support-shell-lab-a1.pem new file mode 100644 index 00000000000..9c6fb099b7f --- /dev/null +++ b/smf/sled-agent/non-gimlet/support-shell-lab-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBnTCCAUSgAwIBAgIUYUmMPRqGDZzMRgqeCasZcAciVcMwCgYIKoZIzj0EAwIw +TTELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +HTAbBgNVBAMMFFN1cHBvcnQgU2hlbGwgTGFiIEExMCIYDzIwMjYxMTA1MTYxNzIw +WhgPMjEyNjEwMTIxNjE3MjBaME0xCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZPeGlk +ZSBDb21wdXRlciBDb21wYW55MR0wGwYDVQQDDBRTdXBwb3J0IFNoZWxsIExhYiBB +MTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJBveaz9eWJ6WTVIGzrkuSy6FiyV +jzTlW3J2Nn83C+V2at7w1ZR6SQhqh72iTUHQdFmoMfbVD/g5zvr7I6tgfMgwCgYI +KoZIzj0EAwIDRwAwRAIgXRsjYcYmwgRdrBLvGyslg5sPtJHR7Aan3IkIxSwYaHUC +IB8Jpc37a3S8RFcMHwFOuZUaRLPVjPNztT3rHCmJVQnE +-----END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/support-shell-production-a1.pem b/smf/sled-agent/non-gimlet/support-shell-production-a1.pem new file mode 100644 index 00000000000..ca3189cdbd0 --- /dev/null +++ b/smf/sled-agent/non-gimlet/support-shell-production-a1.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBrDCCAVKgAwIBAgIUabiqiIIFqIh5XY3mMFSJUC8yjicwCgYIKoZIzj0EAwIw +VDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx +JDAiBgNVBAMMG1N1cHBvcnQgU2hlbGwgUHJvZHVjdGlvbiBBMTAiGA8yMDI2MTEw +NTE2MjEwM1oYDzIxMjYxMDEyMTYyMTAzWjBUMQswCQYDVQQGEwJVUzEfMB0GA1UE +CgwWT3hpZGUgQ29tcHV0ZXIgQ29tcGFueTEkMCIGA1UEAwwbU3VwcG9ydCBTaGVs +bCBQcm9kdWN0aW9uIEExMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAER0R2J/71 +7Ok2C9o3mspibhnYdOEfYwpV6ttuCOsOg1J4btV/rTiw8mf1RhNVJlvajDVyvYlZ +y9M97jn08z+X4jAKBggqhkjOPQQDAgNIADBFAiAQAKpYlsB7/VE9VHkI/VXSa/mq +iIiQ2DGOAhkmj3T0xgIhAJtZE+9HiCbjWvVYWb9VrIUak52BFq3pLdOso7efEl1d +-----END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem b/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem deleted file mode 100644 index 5205bb84253..00000000000 --- a/smf/sled-agent/non-gimlet/sush-sandbox-s1.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBdzCCAR2gAwIBAgIUGrborM2Ih3cSKNzLiDuyTcLdcYQwCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwOTAxMjMy -NDA3WhgPOTk5OTEyMzEyMzU5NTlaMCMxITAfBgNVBAMMGFN1cHBvcnQgU2hlbGwg -U2FuZGJveCBTMTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLkzaBZtSnOY3kkl -6UeIFJAWD/m9VGCVVrZn693E8Wu+6iJaIZKRrzbTpp6JNa3rUQQBM5uoowqozZHC -tKZ8NhkwCgYIKoZIzj0EAwIDSAAwRQIgcpeZcealhiR1w5wmhulT+mQeI0xWj8AG -Njr5LYG1W10CIQCQ29iC9RB+jD7jLJuMWioRf9g3q3sH5letjZ5SqdcJYw== ------END CERTIFICATE----- diff --git a/smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem b/smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem deleted file mode 100644 index 17b6a761ef2..00000000000 --- a/smf/sled-agent/non-gimlet/sush-sandbox-untrusted.pem +++ /dev/null @@ -1,11 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBpTCCAUqgAwIBAgIUcaflT3yMc6uPJUFaRV5NZtUdXi0wCgYIKoZIzj0EAwIw -UDELMAkGA1UEBhMCVVMxHzAdBgNVBAoMFk94aWRlIENvbXB1dGVyIENvbXBhbnkx -IDAeBgNVBAMMF1N1cHBvcnQgU2hlbGwgUHJvdG90eXBlMCIYDzIwMjYwMTI3MTk0 -NzU2WhgPOTk5OTEyMzEyMzU5NTlaMFAxCzAJBgNVBAYTAlVTMR8wHQYDVQQKDBZP -eGlkZSBDb21wdXRlciBDb21wYW55MSAwHgYDVQQDDBdTdXBwb3J0IFNoZWxsIFBy -b3RvdHlwZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABP5+eaTZnpF4tghwJolg -m8te/yAfAnkgzOV1FJiH0nauoM8gsyoiVAu9CwlHNnivju6TslFFyxBlW5h3eMnJ -ULwwCgYIKoZIzj0EAwIDSQAwRgIhAIqvomz/EXcZGCti2QwHkdPOMfidDw02/Rwk -t3x+5DsfAiEAlEeWgVPiLcseslP86MJpLvVPFFO3S08IODCNgOUh7tM= ------END CERTIFICATE----- From 40d6e40334fdcb3014086d15aab1c3327e5e096a Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 6 Nov 2026 10:58:03 -0700 Subject: [PATCH 36/38] Listen only on the bootstrap network Using the underlay network at all was only necessary with an old design that involved Nexus talking directly to the servers (since Nexus isn't on the bootstrap network). But we ended up tunneling connections from Nexus through the switch zone proxy instead (which is on the boostrap network), so there is no need to listen on the underlay. Also cleaned up and cut some related comments. Co-authored-by: Claude --- sled-agent/src/long_running_tasks.rs | 5 -- sled-agent/src/server.rs | 31 +++--------- sled-agent/src/sush.rs | 75 +++++++++++----------------- 3 files changed, 37 insertions(+), 74 deletions(-) diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs index cbd1671cdba..64f380eef00 100644 --- a/sled-agent/src/long_running_tasks.rs +++ b/sled-agent/src/long_running_tasks.rs @@ -215,11 +215,6 @@ pub async fn spawn_all_longrunning_tasks( let health_monitor = spawn_health_monitor_tasks(log).await; - // Because sush must work even when the control plane doesn't, we start it - // before waiting for rack membership. Initially, the sush API is served on - // the bootstrap network. Once the sled is told its underlay address, - // `crate::server::Server::start` will also serve the sush API on the - // underlay network. let sush = match &config.sush { Some(sush_config) => { info!(log, "Starting sush tasks"); diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 1882a6a8b48..f5bdcea491e 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -20,7 +20,6 @@ use slog::Logger; use slog_error_chain::InlineErrorChain; use std::net::SocketAddr; use std::sync::Arc; -use sush_server::JobManager; /// Packages up a [`SledAgent`], running the sled agent API under a Dropshot /// server wired up to the sled agent @@ -28,8 +27,8 @@ pub struct Server { /// Dropshot server for the API. http_server: dropshot::HttpServer, - /// The Support Shell's task handles and API server, if it is running. - sush: Option<(SushHandles, dropshot::HttpServer>)>, + /// The Support Shell's task handles, if it is running. + sush: Option, } impl Server { @@ -64,7 +63,7 @@ impl Server { let nexus_client = make_nexus_client(&log, resolver); - let sush_handles = long_running_tasks_handles.sush.clone(); + let sush = long_running_tasks_handles.sush.clone(); let sled_agent = SledAgent::new( &config, log.clone(), @@ -97,18 +96,6 @@ impl Server { .start() .map_err(|error| format!("initializing server: {}", error))?; - // Now that this sled knows its underlay address, sush can serve - // its API there, too. - let sush = sush_handles.and_then(|handles| { - match handles.start_api(*sled_address.ip()) { - Ok(server) => Some((handles, server)), - Err(error) => { - warn!(log, "failed to start sush server"; "error" => #%error); - None - } - } - }); - Ok(Server { http_server, sush }) } @@ -126,13 +113,9 @@ impl Server { } pub async fn close(self) -> Result<(), String> { - let sush = match self.sush { - Some((handles, server)) => { - handles.shutdown(); - server.close().await - } - None => Ok(()), - }; - self.http_server.close().await.and(sush) + if let Some(sush) = &self.sush { + sush.shutdown(); + } + self.http_server.close().await } } diff --git a/sled-agent/src/sush.rs b/sled-agent/src/sush.rs index 7cb1572fa9d..0a4f8110cbe 100644 --- a/sled-agent/src/sush.rs +++ b/sled-agent/src/sush.rs @@ -4,50 +4,34 @@ //! The Support Shell (`sush`) server that runs in the global zone. //! -//! See RFD 620. This server runs jobs targeted at this sled on behalf of -//! Oxide support. A job arrives signed, and this sled runs it only when -//! the signature chains to one of the trust anchors in its config; see -//! `trust_anchors` for which ones. -//! The server is deliberately independent of the rest of the control plane, -//! because it is one of the tools you reach for when the control plane is -//! the thing that is broken. It therefore starts during bootstrap, before -//! this sled knows whether it is part of a rack at all. +//! See [RFD 620]. The server runs jobs targeted at this sled on behalf of +//! Oxide support. Jobs are authorized by signatures, and the server runs +//! a job only when its signature validates against one of the trust +//! anchors in its config; see `trust_anchors` for how those are selected. //! -//! Startup is in two parts, because its inputs become available at different -//! times: +//! Jobs are distributed among servers using a gossip network implemented +//! by the [rumors] library, running over sprockets on the bootstrap network. +//! _Peers_ (servers) exchange messages that are causally ordered; the set +//! of causally related messages sent between all peers in a network is +//! called a _universe_. Various pieces of universe state are persisted +//! to the M.2s using a [sush locker], described in the _Storage_ section +//! of [RFD 620]. This is similar to an [`omicron_ledger::Ledger`], which +//! also holds records across M.2s, but handles disagreements between the +//! two drives differently. //! -//! * [`spawn_sush_tasks`] builds the job manager as soon as we know our -//! baseboard, and is called from `long_running_tasks`. It serves the -//! API on the bootstrap network, so the switch zone proxy can reach -//! this sled before RSS runs. -//! * [`SushHandles::start_api`] serves the API again on the underlay, -//! and is called from `server` once this sled has been told its -//! underlay address. -//! -//! Gossip runs over sprockets on the bootstrap network. A job may be -//! addressed to a single sled, but the messages carrying jobs, -//! sessions, and events spread to every sled. The set of causally -//! related messages is called a _universe_: peers in the same universe -//! can gossip and converge on its contents; peers in different -//! universes cannot gossip at all. Every identity in a universe -//! descends from a single seed. On the first gossip sync, a deterministic -//! rule decides whose universe wins; the loser joins the winner's -//! universe and adopts a slice of its identity space. A restarted sled -//! therefore rejoins the rack's universe and replays what it missed -//! without re-executing previously run jobs. Each sled stores its identity in -//! a record called a _bookmark_, which is persisted to disk in order to survive -//! the sled restarting. -//! -//! The bookmark, like every record sush must trust across reboots, lives -//! in the [sush locker], described in the [storage section of RFD 620]. -//! This is similar to the [`omicron_ledger::Ledger`], which also holds -//! records across both M.2s, but handles disagreements between the two -//! drives differently. +//! The gossip network is invisible to clients, which speak only the +//! `sush` HTTP API; the server relays requests and events. That API is +//! also served on the bootstrap network only; clients, including those +//! tunneled through Nexus, reach it through a small proxy in the switch +//! zone. It is currently served over unencrypted HTTP, pending intra-rack +//! mTLS support. //! +//! [RFD 620]: +//! https://rfd.shared.oxide.computer/rfd/0620 +//! [rumors]: +//! https://github.com/oxidecomputer/rumors //! [sush locker]: //! https://github.com/oxidecomputer/sush/blob/main/server/src/locker.rs -//! [storage section of RFD 620]: -//! https://rfd.shared.oxide.computer/rfd/0620#_storage use crate::config::SushConfig; use anyhow::{Context, ensure}; @@ -122,7 +106,7 @@ pub struct SushHandles { impl SushHandles { /// Start serving the Support Shell API at `ip`. - pub fn start_api( + fn start_api( &self, ip: Ipv6Addr, ) -> anyhow::Result>> { @@ -241,6 +225,7 @@ pub async fn spawn_sush_tasks( } } }); + let listen_addr = SocketAddrV6::new(bootstrap_ip, SUSH_GOSSIP_PORT, 0, 0); let (universe, linked) = match spawn_gossip( &log, @@ -258,13 +243,13 @@ pub async fn spawn_sush_tasks( Err(err) => { warn!( log, - "gossip disabled, this sled serves local jobs only"; + "gossip disabled, this sled will run local jobs only"; "error" => InlineErrorChain::new(&err), ); - // This seed never reaches a peer, so it is not worth - // storing. The null locker stores nothing, leaving the - // bookmark on the M.2s untouched for the next boot. - let seed = seed_gossip(&log, &Locker::null()).await; + + // Don't persist an isolated identity. + let locker = Locker::null(); + let seed = seed_gossip(&log, &locker).await; (Universe::isolated(seed.into_rumors()), LinkedBaseboards::lonely()) } }; From 289dbecf57f7c1a7a4d2aa3cdf27039321655954 Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Fri, 6 Nov 2026 13:10:06 -0700 Subject: [PATCH 37/38] Update sush to latest main Co-authored-by: Claude --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2232937a5f6..46f5c15a16a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16052,7 +16052,7 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "sush-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=b72da8c514decf9f8d8e57f49463b8d2f93333be#b72da8c514decf9f8d8e57f49463b8d2f93333be" +source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" dependencies = [ "chrono", "dropshot 0.18.0", @@ -16069,7 +16069,7 @@ dependencies = [ [[package]] name = "sush-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=b72da8c514decf9f8d8e57f49463b8d2f93333be#b72da8c514decf9f8d8e57f49463b8d2f93333be" +source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" dependencies = [ "bytes", "bytesize", @@ -16104,7 +16104,7 @@ dependencies = [ [[package]] name = "sush-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=b72da8c514decf9f8d8e57f49463b8d2f93333be#b72da8c514decf9f8d8e57f49463b8d2f93333be" +source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" dependencies = [ "async-trait", "atomicwrites", diff --git a/Cargo.toml b/Cargo.toml index e05355aef3f..decddd5812b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -876,9 +876,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" -sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "b72da8c514decf9f8d8e57f49463b8d2f93333be" } -sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "b72da8c514decf9f8d8e57f49463b8d2f93333be" } -sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "b72da8c514decf9f8d8e57f49463b8d2f93333be", features = ["embedded"] } +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] } From aed01815d1465d8ff204001683593de09b2c38ea Mon Sep 17 00:00:00 2001 From: Alex Plotnick Date: Sat, 7 Nov 2026 14:38:48 -0700 Subject: [PATCH 38/38] Update sush to latest main Co-authored-by: Claude --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46f5c15a16a..1b15a516726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16052,7 +16052,7 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "sush-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" +source = "git+https://github.com/oxidecomputer/sush?rev=7ab073efccf0d2ef54a316a00101586547b931d6#7ab073efccf0d2ef54a316a00101586547b931d6" dependencies = [ "chrono", "dropshot 0.18.0", @@ -16069,7 +16069,7 @@ dependencies = [ [[package]] name = "sush-common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" +source = "git+https://github.com/oxidecomputer/sush?rev=7ab073efccf0d2ef54a316a00101586547b931d6#7ab073efccf0d2ef54a316a00101586547b931d6" dependencies = [ "bytes", "bytesize", @@ -16104,7 +16104,7 @@ dependencies = [ [[package]] name = "sush-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/sush?rev=26854d1b85ee3664eac26e73833b2c70161ccd52#26854d1b85ee3664eac26e73833b2c70161ccd52" +source = "git+https://github.com/oxidecomputer/sush?rev=7ab073efccf0d2ef54a316a00101586547b931d6#7ab073efccf0d2ef54a316a00101586547b931d6" dependencies = [ "async-trait", "atomicwrites", diff --git a/Cargo.toml b/Cargo.toml index decddd5812b..80da6102bcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -876,9 +876,9 @@ subtle = "2.6.1" supports-color = "3.0.2" support-bundle-collection = { path = "support-bundle-collection" } support-bundle-viewer = "0.1.2" -sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52" } -sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52" } -sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "26854d1b85ee3664eac26e73833b2c70161ccd52", features = ["embedded"] } +sush-api = { git = "https://github.com/oxidecomputer/sush", rev = "7ab073efccf0d2ef54a316a00101586547b931d6" } +sush-common = { git = "https://github.com/oxidecomputer/sush", rev = "7ab073efccf0d2ef54a316a00101586547b931d6" } +sush-server = { git = "https://github.com/oxidecomputer/sush", rev = "7ab073efccf0d2ef54a316a00101586547b931d6", features = ["embedded"] } swrite = "0.1.0" sync-ptr = "0.1.4" libsw = { version = "3.5.0", features = ["tokio"] }