Build internal HTTP clients without loading TLS roots

clients/ddm-admin-client/src/lib.rs

@@ -14,6 +14,7 @@ use either::Either;
 use omicron_common::address::BOOTSTRAP_PREFIX;
 use omicron_common::address::BOOTSTRAP_SLED_SUBNET_PREFIX_LENGTH;
 use omicron_common::address::DDMD_PORT;
+use omicron_common::http_client::internal_http_client_builder;
 use oxnet::Ipv6Net;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
@@ -54,7 +55,7 @@ impl Client {
         let log =
             log.new(slog::o!("DdmAdminClient" => SocketAddr::V6(ddmd_addr)));
 
-        let inner = reqwest::ClientBuilder::new()
+        let inner = internal_http_client_builder()
             .connect_timeout(dur)
             .timeout(dur)
             .build()?;

cockroach-admin/src/http_entrypoints.rs

@@ -12,6 +12,7 @@ use dropshot::HttpResponseOk;
 use dropshot::HttpResponseUpdatedNoContent;
 use dropshot::RequestContext;
 use dropshot::TypedBody;
+use omicron_common::http_client::internal_http_client_builder;
 use slog::info;
 use slog_error_chain::InlineErrorChain;
 use std::sync::Arc;
@@ -86,7 +87,9 @@ impl CockroachAdminApi for CockroachAdminImpl {
             ctx.cockroach_cli().cockroach_http_address();
         let url = format!("http://{}/_status/vars", cockroach_http_address);
 
-        let client = reqwest::Client::new();
+        let client = internal_http_client_builder()
+            .build()
+            .expect("building a client without TLS roots does no I/O");
         let response = client.get(&url).send().await.map_err(|e| {
             HttpError::for_internal_error(format!(
                 "Failed to proxy to CockroachDB: {}",
@@ -128,7 +131,9 @@ impl CockroachAdminApi for CockroachAdminImpl {
             ctx.cockroach_cli().cockroach_http_address();
         let url = format!("http://{}/_status/nodes", cockroach_http_address);
 
-        let client = reqwest::Client::new();
+        let client = internal_http_client_builder()
+            .build()
+            .expect("building a client without TLS roots does no I/O");
         let response = client.get(&url).send().await.map_err(|e| {
             HttpError::for_internal_error(format!(
                 "Failed to proxy to CockroachDB: {}",

cockroach-metrics/Cargo.toml

@@ -10,6 +10,7 @@ chrono.workspace = true
 cockroach-admin-client.workspace = true
 cockroach-admin-types.workspace = true
 futures.workspace = true
+omicron-common.workspace = true
 parallel-task-set.workspace = true
 reqwest.workspace = true
 serde_json.workspace = true

cockroach-metrics/src/lib.rs

@@ -12,6 +12,7 @@ use chrono::{DateTime, Utc};
 use cockroach_admin_client::Client;
 use cockroach_admin_types::node::InternalNodeId;
 use futures::stream::{FuturesUnordered, StreamExt};
+use omicron_common::http_client::internal_http_client_builder;
 use parallel_task_set::ParallelTaskSet;
 use serde::{Deserialize, Serialize};
 use slog::{Logger, debug, warn};
@@ -36,7 +37,7 @@ impl CockroachAdminClient {
     ///
     /// "timeout" is used as both a connection and request timeout duration.
     fn new(log: Logger, address: SocketAddr, timeout: Duration) -> Self {
-        let reqwest_client = reqwest::ClientBuilder::new()
+        let reqwest_client = internal_http_client_builder()
             .connect_timeout(timeout)
             .timeout(timeout)
             .build()

common/src/http_client.rs

@@ -0,0 +1,97 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Construction of HTTP clients for talking to other control plane services.
+//!
+//! By default, building a [`reqwest::Client`] loads the system's TLS root
+//! certificates from disk, and fails if none can be read (for example, if the
+//! process has run out of file descriptors). Services within the control plane
+//! talk to each other over plain HTTP, so they have no use for those
+//! certificates.
+//!
+//! See <https://github.com/oxidecomputer/omicron/issues/10871>.
+
+use std::time::Duration;
+
+/// The connect and request timeout applied by a progenitor-generated
+/// `Client::new`.
+const PROGENITOR_DEFAULT_TIMEOUT: Duration = Duration::from_secs(15);
+
+/// Returns a [`reqwest::ClientBuilder`] for plain-HTTP communication with
+/// other control plane services.
+///
+/// The resulting client trusts no TLS root certificates: requests to `https`
+/// URLs will fail certificate verification. Clients that need to speak TLS
+/// (e.g., to services outside the rack) should not use this builder.
+pub fn internal_http_client_builder() -> reqwest::ClientBuilder {
+    reqwest::ClientBuilder::new().tls_certs_only([])
+}
+
+/// Returns a [`reqwest::Client`] for plain-HTTP communication with other
+/// control plane services, configured with the same timeouts as a
+/// progenitor-generated `Client::new`.
+///
+/// Pass this to a generated client's `new_with_client` in place of calling its
+/// `new`, which builds a client that loads TLS root certificates from disk.
+pub fn internal_http_client() -> reqwest::Client {
+    internal_http_client_builder()
+        .connect_timeout(PROGENITOR_DEFAULT_TIMEOUT)
+        .timeout(PROGENITOR_DEFAULT_TIMEOUT)
+        .build()
+        .expect("building a client without TLS roots does no I/O")
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::fs::File;
+
+    // This test exhausts the process's file descriptors, so it relies on
+    // nextest running each test in its own process.
+    #[test]
+    fn internal_http_client_builds_without_file_descriptors() {
+        // Lower the soft limit on open files, so that we don't need to open
+        // very many to run out.
+        let mut limit = libc::rlimit { rlim_cur: 0, rlim_max: 0 };
+        // SAFETY: `limit` is a valid, writable `rlimit`.
+        let rc = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) };
+        assert_eq!(rc, 0, "getrlimit: {}", std::io::Error::last_os_error());
+        limit.rlim_cur = limit.rlim_cur.min(64);
+        // SAFETY: `limit` is a valid `rlimit`.
+        let rc = unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) };
+        assert_eq!(rc, 0, "setrlimit: {}", std::io::Error::last_os_error());
+
+        let mut held = Vec::new();
+        let err = loop {
+            match File::open("/dev/null") {
+                Ok(f) => held.push(f),
+                Err(e) => break e,
+            }
+        };
+        assert_eq!(
+            err.raw_os_error(),
+            Some(libc::EMFILE),
+            "expected to run out of file descriptors, got: {err}"
+        );
+
+        // Panics if the client cannot be built.
+        let _ = internal_http_client();
+
+        // On these platforms, reqwest's default client loads TLS roots from
+        // disk, so it should fail to build. If this starts succeeding, the
+        // test above no longer demonstrates anything about TLS roots.
+        //
+        // This check is skipped on macOS, where building the default client
+        // does no I/O: the platform verifier consults the system trust store
+        // only when verifying a certificate.
+        #[cfg(any(target_os = "linux", target_os = "illumos"))]
+        reqwest::ClientBuilder::new().build().expect_err(
+            "expected a default reqwest client to fail to build without \
+             file descriptors (has reqwest stopped loading TLS roots when \
+             building a client?)",
+        );
+
+        drop(held);
+    }
+}

common/src/lib.rs

@@ -23,6 +23,7 @@ pub mod api;
 pub mod backoff;
 pub mod cmd;
 pub mod disk;
+pub mod http_client;
 pub mod policy;
 pub mod resolvable_files;
 pub mod snake_case_result;

installinator/src/artifact.rs

@@ -2,6 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+use omicron_common::http_client::internal_http_client_builder;
 use std::{net::SocketAddr, time::Duration};
 
 use anyhow::{Context, Result, bail};
@@ -118,7 +119,7 @@ impl ArtifactClient {
         // by the fetch loop. (Why is the read timeout handled by the fetch
         // loop? So that it can also apply to the mock peer backend, and logic
         // shared across both.)
-        let client = reqwest::ClientBuilder::new()
+        let client = internal_http_client_builder()
             .connect_timeout(Duration::from_secs(5))
             .timeout(Duration::from_secs(5 * 60))
             .build()

nexus/inventory/src/collector.rs

@@ -24,6 +24,8 @@ use nexus_types::inventory::SpType;
 use nexus_types::inventory::TimeSync;
 use omicron_cockroach_metrics::CockroachClusterAdminClient;
 use omicron_common::address::NTP_ADMIN_PORT;
+use omicron_common::http_client::internal_http_client;
+use omicron_common::http_client::internal_http_client_builder;
 use omicron_uuid_kinds::OmicronZoneUuid;
 use parallel_task_set::ParallelTaskSet;
 use sled_agent_types::disk::M2Slot;
@@ -528,7 +530,14 @@ impl<'a> Collector<'a> {
                 let url = format!("http://{addr}");
                 let log = self.log.new(o!("ntp_admin_url" => url.clone()));
 
-                (cfg.id, ntp_admin_client::Client::new(&url, log))
+                (
+                    cfg.id,
+                    ntp_admin_client::Client::new_with_client(
+                        &url,
+                        internal_http_client(),
+                        log,
+                    ),
+                )
             })
             .collect();
 
@@ -657,7 +666,14 @@ impl<'a> Collector<'a> {
                 let url = format!("http://{http_address}");
                 let log = self.log.new(o!("internal_dns_url" => url.clone()));
 
-                (cfg.id, dns_service_client::Client::new(&url, log))
+                (
+                    cfg.id,
+                    dns_service_client::Client::new_with_client(
+                        &url,
+                        internal_http_client(),
+                        log,
+                    ),
+                )
             })
             .collect();
 
@@ -709,7 +725,7 @@ async fn collect_one_sled_agent(
         "sled_agent_url" => url
     );
 
-    let reqwest_client = reqwest::ClientBuilder::new()
+    let reqwest_client = internal_http_client_builder()
         .connect_timeout(SLED_AGENT_TIMEOUT)
         .timeout(SLED_AGENT_TIMEOUT)
         .build()

nexus/metrics-producer-gc/src/lib.rs

@@ -20,6 +20,7 @@ use nexus_db_queries::db::identity::Asset;
 use nexus_db_queries::db::model::OximeterInfo;
 use nexus_db_queries::db::model::ProducerEndpoint;
 use omicron_common::api::external::Error as DbError;
+use omicron_common::http_client::internal_http_client;
 use oximeter_client::Client as OximeterClient;
 use slog::Logger;
 use slog::info;
@@ -173,8 +174,11 @@ impl ExpiredProducers {
             let client_log =
                 opctx.log.new(o!("oximeter-collector" => info.id.to_string()));
             let address = SocketAddr::new(info.ip.ip(), *info.port);
-            let client =
-                OximeterClient::new(&format!("http://{address}"), client_log);
+            let client = OximeterClient::new_with_client(
+                &format!("http://{address}"),
+                internal_http_client(),
+                client_log,
+            );
             entry.insert(client);
         }
 

nexus/mgs-updates/src/artifacts.rs

@@ -5,6 +5,7 @@
 //! Makes artifact contents available for use in updates
 
 use futures::TryStreamExt;
+use omicron_common::http_client::internal_http_client;
 use sha2::{Digest, Sha256};
 use slog::o;
 use std::sync::atomic::AtomicUsize;
@@ -91,7 +92,11 @@ impl ArtifactCache {
             let addr = addresses[idx % addresses.len()];
             let url = format!("http://{}", addr.address);
             let log = self.log.new(o!("repo_depot_url" => url.clone()));
-            Ok(repo_depot_client::Client::new(&url, log))
+            Ok(repo_depot_client::Client::new_with_client(
+                &url,
+                internal_http_client(),
+                log,
+            ))
         }
     }
 }

nexus/mgs-updates/src/driver_update.rs

@@ -20,6 +20,7 @@ use nexus_types::deployment::PendingMgsUpdateDetails;
 use nexus_types::internal_api::views::UpdateAttemptStatus;
 use nexus_types::internal_api::views::UpdateCompletedHow;
 use nexus_types::inventory::SpType;
+use omicron_common::http_client::internal_http_client_builder;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::SpUpdateUuid;
 use qorb::resolver::AllBackends;
@@ -204,7 +205,7 @@ fn make_mgs_clients(backends: &AllBackends, log: &slog::Logger) -> MgsClients {
         // because we've seen sidecars take 20+ seconds to reset). We should
         // therefore wait at least as long as its timeouts; we'll add a buffer
         // here to leave plenty of time for minor weather between us and MGS.
-        let client = reqwest::ClientBuilder::new()
+        let client = internal_http_client_builder()
             .connect_timeout(Duration::from_secs(75))
             .timeout(Duration::from_secs(75))
             .build()

nexus/mgs-updates/src/host_phase1_updater.rs

@@ -142,6 +142,7 @@ use gateway_client::types::SpComponentFirmwareSlot;
 use nexus_types::deployment::PendingMgsUpdate;
 use nexus_types::deployment::PendingMgsUpdateHostPhase1Details;
 use nexus_types::inventory::SpType;
+use omicron_common::http_client::internal_http_client;
 use sled_agent_client::Client as SledAgentClient;
 use sled_agent_types::disk::M2Slot;
 use sled_agent_types::inventory::BootPartitionContents;
@@ -345,8 +346,11 @@ impl ReconfiguratorHostPhase1Updater {
         log: &Logger,
     ) -> Result<BootPartitionContents, PrecheckError> {
         let address = self.details.sled_agent_address;
-        let sled_agent =
-            SledAgentClient::new(&format!("http://{address}"), log.clone());
+        let sled_agent = SledAgentClient::new_with_client(
+            &format!("http://{address}"),
+            internal_http_client(),
+            log.clone(),
+        );
         let sled_inventory = sled_agent
             .inventory()
             .await

nexus/networking/src/gateway_client.rs

@@ -3,6 +3,7 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use omicron_common::backoff::{self, BackoffError};
+use omicron_common::http_client::internal_http_client;
 use omicron_uuid_kinds::RackUuid;
 use parallel_task_set::ParallelTaskSet;
 use slog::{Logger, o};
@@ -31,7 +32,11 @@ impl GatewayClient {
         let url = format!("http://{addr}");
         let log = log
             .new(o!("gateway_url" => url.clone(), "addr" => addr.to_string()));
-        let client = gateway_client::Client::new(&url, log);
+        let client = gateway_client::Client::new_with_client(
+            &url,
+            internal_http_client(),
+            log,
+        );
         GatewayClient { addr, client }
     }
 

nexus/networking/src/sled_client.rs

@@ -10,6 +10,7 @@ use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::LookupResult;
+use omicron_common::http_client::internal_http_client_builder;
 use omicron_uuid_kinds::RackUuid;
 use omicron_uuid_kinds::SledUuid;
 use sled_agent_client::Client as SledAgentClient;
@@ -29,7 +30,7 @@ pub fn sled_lookup<'a>(
 
 pub fn default_reqwest_client_builder() -> reqwest::ClientBuilder {
     let dur = std::time::Duration::from_secs(60);
-    reqwest::ClientBuilder::new().connect_timeout(dur).timeout(dur)
+    internal_http_client_builder().connect_timeout(dur).timeout(dur)
 }
 
 pub async fn sled_client(