Add support for s3-compatible storage

Author: travisby

src/server/cloud/aws.rs

@@ -13,7 +13,7 @@ use aws_sdk_s3::{
 use std::future::Future;
 use tokio::runtime::Runtime;
 
-/// A [`Service`] implementation based on the Google Cloud Storage service.
+/// A [`Service`] implementation based on the AWS Simple Storage Service.
 pub(in crate::server) struct AwsService {
     client: s3::Client,
     rt: Runtime,
@@ -63,9 +63,11 @@ pub enum AwsCredentials {
 
 impl AwsService {
     pub(in crate::server) fn new(
-        region: String,
+        region: Option<String>,
         bucket: String,
         creds: AwsCredentials,
+        endpoint_url: Option<String>,
+        force_path_style: bool,
     ) -> Result<Self> {
         let rt = Runtime::new()?;
 
@@ -92,13 +94,37 @@ impl AwsService {
                         // Just use the default.
                     }
                 }
-                config_provider
-                    .region(RegionProviderChain::first_try(Region::new(region)))
-                    .load()
-                    .await
+                // This will:
+                // 1. If a region is set, use it
+                // 2. if No region is set, follow the AWS default provider chain
+                //    (https://docs.aws.amazon.com/sdk-for-rust/latest/dg/region.html)
+                // 3. If no region is discovered, hardcode to "us-east-1"
+                //
+                // If there's a region specified we will always prefer that
+                // next, the default provider chain will look at things like AWS_REGION environment
+                // variables, the profile file, etc.
+                //
+                // we provide the hardcoded fallback because a region MUST be set
+                // but, a region being set does _not_ make sense if endpoint_url is set, because
+                // the endpoint URL would include a region.
+                // realistically, endpoint_url is more useful for S3-compatible services
+                // and would not use a separate region in addition to endpoint_url.
+                config_provider = config_provider.region(
+                    RegionProviderChain::first_try(region.map(Region::new))
+                        .or_default_provider()
+                        .or_else(Region::new("us-east-1")),
+                );
+                match endpoint_url {
+                    Some(url) => config_provider = config_provider.endpoint_url(url),
+                    None => {}
+                }
+                config_provider.load().await
             });
 
-        let client = s3::client::Client::new(&config);
+        let s3_config = aws_sdk_s3::config::Builder::from(&config)
+            .force_path_style(force_path_style)
+            .build();
+        let client = aws_sdk_s3::Client::from_conf(s3_config);
         Ok(Self { client, rt, bucket })
     }
 
@@ -405,12 +431,14 @@ mod tests {
 
         Some(
             AwsService::new(
-                region,
+                Some(region),
                 bucket,
                 AwsCredentials::AccessKey {
                     access_key_id,
                     secret_access_key,
                 },
+                None,
+                false,
             )
             .unwrap(),
         )

src/server/config.rs

@@ -79,7 +79,7 @@ pub enum ServerConfig {
     #[cfg(feature = "server-aws")]
     Aws {
         /// Region in which the bucket is located.
-        region: String,
+        region: Option<String>,
         /// Bucket in which to store the task data.
         ///
         /// This bucket must not be used for any other purpose. No special bucket configuration is
@@ -90,6 +90,12 @@ pub enum ServerConfig {
         /// Private encryption secret used to encrypt all data sent to the server.  This can
         /// be any suitably un-guessable string of bytes.
         encryption_secret: Vec<u8>,
+        // endpoint_url is an optional URL to specify the hostname of an s3-compatible service.
+        // When endpoint_url is used, region is ignored by the underlying S3 client.
+        endpoint_url: Option<String>,
+        // force_path_style is used to force the S3 client to use path-style URLs instead of
+        // virtual-hosted-style (subdomain) URLs for the bucket.
+        force_path_style: bool,
     },
 }
 
@@ -120,8 +126,10 @@ impl ServerConfig {
                 bucket,
                 credentials,
                 encryption_secret,
+                endpoint_url,
+                force_path_style,
             } => Box::new(CloudServer::new(
-                AwsService::new(region, bucket, credentials)?,
+                AwsService::new(region, bucket, credentials, endpoint_url, force_path_style)?,
                 encryption_secret,
             )?),
         })