[nexus] Create a longer-lived connection from Nexus -> Sled Agent (#8149)

Fixes #8144

Author: smklein

nexus/networking/src/sled_client.rs

@@ -25,29 +25,49 @@ pub fn sled_lookup<'a>(
     Ok(sled)
 }
 
+pub fn default_reqwest_client_builder() -> reqwest::ClientBuilder {
+    let dur = std::time::Duration::from_secs(60);
+    reqwest::ClientBuilder::new().connect_timeout(dur).timeout(dur)
+}
+
 pub async fn sled_client(
     datastore: &DataStore,
     lookup_opctx: &OpContext,
     sled_id: Uuid,
     log: &Logger,
+) -> Result<SledAgentClient, Error> {
+    let client = default_reqwest_client_builder().build().unwrap();
+    sled_client_ext(datastore, lookup_opctx, sled_id, log, client).await
+}
+
+pub async fn sled_client_ext(
+    datastore: &DataStore,
+    lookup_opctx: &OpContext,
+    sled_id: Uuid,
+    log: &Logger,
+    client: reqwest::Client,
 ) -> Result<SledAgentClient, Error> {
     let (.., sled) =
         sled_lookup(datastore, lookup_opctx, sled_id)?.fetch().await?;
 
-    Ok(sled_client_from_address(sled_id, sled.address(), log))
+    Ok(sled_client_from_address_ext(sled_id, sled.address(), log, client))
 }
 
 pub fn sled_client_from_address(
     sled_id: Uuid,
     address: SocketAddrV6,
     log: &Logger,
+) -> SledAgentClient {
+    let client = default_reqwest_client_builder().build().unwrap();
+    sled_client_from_address_ext(sled_id, address, log, client)
+}
+
+pub fn sled_client_from_address_ext(
+    sled_id: Uuid,
+    address: SocketAddrV6,
+    log: &Logger,
+    client: reqwest::Client,
 ) -> SledAgentClient {
     let log = log.new(o!("SledAgent" => sled_id.to_string()));
-    let dur = std::time::Duration::from_secs(60);
-    let client = reqwest::ClientBuilder::new()
-        .connect_timeout(dur)
-        .timeout(dur)
-        .build()
-        .unwrap();
     SledAgentClient::new_with_client(&format!("http://{address}"), client, log)
 }

nexus/src/app/sled.rs

@@ -147,6 +147,16 @@ impl super::Nexus {
     pub async fn sled_client(
         &self,
         id: &SledUuid,
+    ) -> Result<Arc<SledAgentClient>, Error> {
+        let client =
+            nexus_networking::default_reqwest_client_builder().build().unwrap();
+        self.sled_client_ext(id, client).await
+    }
+
+    pub async fn sled_client_ext(
+        &self,
+        id: &SledUuid,
+        client: reqwest::Client,
     ) -> Result<Arc<SledAgentClient>, Error> {
         // TODO: We should consider injecting connection pooling here,
         // but for now, connections to sled agents are constructed
@@ -155,11 +165,12 @@ impl super::Nexus {
         // Frankly, returning an "Arc" here without a connection pool is a
         // little silly; it's not actually used if each client connection exists
         // as a one-shot.
-        let client = nexus_networking::sled_client(
+        let client = nexus_networking::sled_client_ext(
             &self.db_datastore,
             &self.opctx_alloc,
             id.into_untyped_uuid(),
             &self.log,
+            client,
         )
         .await?;
         Ok(Arc::new(client))

nexus/src/app/support_bundles.rs

@@ -90,7 +90,20 @@ impl super::Nexus {
             .db_datastore
             .zpool_get_sled_if_in_service(&opctx, bundle.zpool_id.into())
             .await?;
-        let client = self.sled_client(&sled_id).await?;
+
+        let short_timeout = std::time::Duration::from_secs(60);
+        let long_timeout = std::time::Duration::from_secs(3600);
+        let client = nexus_networking::default_reqwest_client_builder()
+            // Continuing to read from the sled agent should happen relatively
+            // quickly.
+            .read_timeout(short_timeout)
+            // However, the bundle itself may be large. As long as we're
+            // continuing to make progress (see: read_timeout) we should be
+            // willing to keep transferring the bundle for a while longer.
+            .timeout(long_timeout)
+            .build()
+            .expect("Failed to build reqwest Client");
+        let client = self.sled_client_ext(&sled_id, client).await?;
 
         // TODO: Use "range"?