diff --git a/Cargo.lock b/Cargo.lock index 1d0bc4529..a5f3c70a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1209,11 +1209,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ - "libc", + "shlex", ] [[package]] diff --git a/crates/sb_fs/Cargo.toml b/crates/sb_fs/Cargo.toml index e73b2322d..f2328f0bc 100644 --- a/crates/sb_fs/Cargo.toml +++ b/crates/sb_fs/Cargo.toml @@ -41,10 +41,10 @@ either.workspace = true tracing.workspace = true rkyv = { workspace = true, features = ["validation"] } -rustls = "0.21" -hyper-proxy = { version = "0.9", default-features = false, features = ["rustls"] } -hyper-rustls = "0.24" -headers = "0.3" +rustls = { version = "0.21", optional = true } +hyper-proxy = { version = "0.9", optional = true, default-features = false, features = ["rustls"] } +hyper-rustls = { version = "0.24", optional = true } +headers = { version = "0.3", optional = true } normalize-path = "0.2" memmap2 = "0.9" aws-sdk-s3 = "1.2" @@ -63,4 +63,12 @@ serial_test.workspace = true dotenvy = "0.15" aws-smithy-runtime = { version = "1.7", features = ["test-util"] } -aws-smithy-runtime-api = { version = "1.7", features = ["test-util"] } \ No newline at end of file +aws-smithy-runtime-api = { version = "1.7", features = ["test-util"] } + +[features] +unsafe-proxy = [ + "dep:rustls", + "dep:hyper-proxy", + "dep:hyper-rustls", + "dep:headers" +] \ No newline at end of file diff --git a/crates/sb_fs/fs/s3_fs.rs b/crates/sb_fs/fs/s3_fs.rs index 408f2aea9..bc485bc62 100644 --- a/crates/sb_fs/fs/s3_fs.rs +++ b/crates/sb_fs/fs/s3_fs.rs @@ -17,6 +17,7 @@ use std::{ time::{Duration, SystemTime, UNIX_EPOCH}, }; +use super::TryNormalizePath; use anyhow::{anyhow, Context}; use aws_config::{retry::RetryConfig, AppName, BehaviorVersion, Region}; use aws_credential_types::{credential_fn::provide_credentials_fn, Credentials}; @@ -44,12 +45,9 @@ use futures::{ stream::FuturesUnordered, AsyncWriteExt, FutureExt, StreamExt, TryFutureExt, }; -use headers::Authorization; -use hyper_proxy::{Intercept, Proxy, ProxyConnector}; -use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; -use hyper_v014::{client::HttpConnector, Uri}; + use memmap2::{MmapOptions, MmapRaw}; -use once_cell::sync::{Lazy, OnceCell}; +use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use tempfile::tempfile; use tokio::{ @@ -58,9 +56,6 @@ use tokio::{ task::JoinError, }; use tracing::{debug, error, info_span, instrument, trace, trace_span, warn, Instrument}; -use url::Url; - -use super::TryNormalizePath; const MIN_PART_SIZE: usize = 1024 * 1024 * 5; @@ -240,9 +235,17 @@ impl S3FsConfig { CLIENT.with(|it| { it.borrow_mut() .get_or_init(|| { - if let Some(proxy_connector) = resolve_proxy_connector() { - HyperClientBuilder::new().build(proxy_connector) - } else { + #[cfg(feature = "unsafe-proxy")] + { + if let Some(proxy_connector) = resolve_proxy_connector() { + HyperClientBuilder::new().build(proxy_connector) + } else { + HyperClientBuilder::new().build_https() + } + } + + #[cfg(not(feature = "unsafe-proxy"))] + { HyperClientBuilder::new().build_https() } }) @@ -251,7 +254,17 @@ impl S3FsConfig { } } -fn resolve_proxy_connector() -> Option>> { +#[cfg(feature = "unsafe-proxy")] +fn resolve_proxy_connector() -> Option< + hyper_proxy::ProxyConnector>, +> { + use headers::Authorization; + use hyper_proxy::{Intercept, Proxy, ProxyConnector}; + use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; + use hyper_v014::{client::HttpConnector, Uri}; + use once_cell::sync::Lazy; + use url::Url; + let proxy_url: Url = std::env::var("HTTPS_PROXY").ok()?.parse().ok()?; let proxy_uri: Uri = std::env::var("HTTPS_PROXY").ok()?.parse().ok()?; let mut proxy = Proxy::new(Intercept::All, proxy_uri);