Skip to content

Commit e975177

Browse files
tustvoldtustvold
authored
ObjectStore WASM32 Support (apache#7226)
* ObjectStore WASM32 Support * Docs * Update .github/workflows/object_store.yml
1 parent bd0c489 commit e975177

File tree

10 files changed

+87
-42
lines changed

10 files changed

+87
-42
lines changed

‎src/aws/builder.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use crate::aws::{
2323
AmazonS3, AwsCredential, AwsCredentialProvider, Checksum, S3ConditionalPut, S3CopyIfNotExists,
2424
STORE,
2525
};
26-
use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider};
26+
use crate::client::{http_connector, HttpConnector, TokenCredentialProvider};
2727
use crate::config::ConfigValue;
2828
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
2929
use base64::prelude::BASE64_STANDARD;
@@ -883,7 +883,9 @@ impl AmazonS3Builder {
883883
self
884884
}
885885

886-
/// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`]
886+
/// The [`HttpConnector`] to use
887+
///
888+
/// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
887889
pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
888890
self.http_connector = Some(Arc::new(connector));
889891
self
@@ -896,9 +898,7 @@ impl AmazonS3Builder {
896898
self.parse_url(&url)?;
897899
}
898900

899-
let http = self
900-
.http_connector
901-
.unwrap_or_else(|| Arc::new(ReqwestConnector::default()));
901+
let http = http_connector(self.http_connector)?;
902902

903903
let bucket = self.bucket_name.ok_or(Error::MissingBucketName)?;
904904
let region = self.region.unwrap_or_else(|| "us-east-1".to_string());

‎src/aws/mod.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,16 @@ mod client;
5858
mod credential;
5959
mod dynamo;
6060
mod precondition;
61+
62+
#[cfg(not(target_arch = "wasm32"))]
6163
mod resolve;
6264

6365
pub use builder::{AmazonS3Builder, AmazonS3ConfigKey};
6466
pub use checksum::Checksum;
6567
pub use dynamo::DynamoCommit;
6668
pub use precondition::{S3ConditionalPut, S3CopyIfNotExists};
69+
70+
#[cfg(not(target_arch = "wasm32"))]
6771
pub use resolve::resolve_bucket_region;
6872

6973
/// This struct is used to maintain the URI path encoding

‎src/azure/builder.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use crate::azure::credential::{
2121
ImdsManagedIdentityProvider, WorkloadIdentityOAuthProvider,
2222
};
2323
use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE};
24-
use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider};
24+
use crate::client::{http_connector, HttpConnector, TokenCredentialProvider};
2525
use crate::config::ConfigValue;
2626
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
2727
use percent_encoding::percent_decode_str;
@@ -889,7 +889,9 @@ impl MicrosoftAzureBuilder {
889889
self
890890
}
891891

892-
/// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`]
892+
/// The [`HttpConnector`] to use
893+
///
894+
/// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
893895
pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
894896
self.http_connector = Some(Arc::new(connector));
895897
self
@@ -907,9 +909,7 @@ impl MicrosoftAzureBuilder {
907909
Arc::new(StaticCredentialProvider::new(credential))
908910
};
909911

910-
let http = self
911-
.http_connector
912-
.unwrap_or_else(|| Arc::new(ReqwestConnector::default()));
912+
let http = http_connector(self.http_connector)?;
913913

914914
let (is_emulator, storage_url, auth, account) = if self.use_emulator.get()? {
915915
let account_name = self

‎src/client/body.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ impl HttpRequestBody {
3939
Self(Inner::Bytes(Bytes::new()))
4040
}
4141

42+
#[cfg(not(target_arch = "wasm32"))]
4243
pub(crate) fn into_reqwest(self) -> reqwest::Body {
4344
match self.0 {
4445
Inner::Bytes(b) => b.into(),

‎src/client/connection.rs

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,14 @@ impl HttpError {
8484
}
8585

8686
pub(crate) fn reqwest(e: reqwest::Error) -> Self {
87+
#[cfg(not(target_arch = "wasm32"))]
88+
let is_connect = || e.is_connect();
89+
#[cfg(target_arch = "wasm32")]
90+
let is_connect = || false;
91+
8792
let mut kind = if e.is_timeout() {
8893
HttpErrorKind::Timeout
89-
} else if e.is_connect() {
94+
} else if is_connect() {
9095
HttpErrorKind::Connect
9196
} else if e.is_decode() {
9297
HttpErrorKind::Decode
@@ -200,6 +205,7 @@ impl HttpClient {
200205
}
201206

202207
#[async_trait]
208+
#[cfg(not(target_arch = "wasm32"))]
203209
impl HttpService for reqwest::Client {
204210
async fn call(&self, req: HttpRequest) -> Result<HttpResponse, HttpError> {
205211
let (parts, body) = req.into_parts();
@@ -227,11 +233,37 @@ pub trait HttpConnector: std::fmt::Debug + Send + Sync + 'static {
227233
/// [`HttpConnector`] using [`reqwest::Client`]
228234
#[derive(Debug, Default)]
229235
#[allow(missing_copy_implementations)]
236+
#[cfg(not(target_arch = "wasm32"))]
230237
pub struct ReqwestConnector {}
231238

239+
#[cfg(not(target_arch = "wasm32"))]
232240
impl HttpConnector for ReqwestConnector {
233241
fn connect(&self, options: &ClientOptions) -> crate::Result<HttpClient> {
234242
let client = options.client()?;
235243
Ok(HttpClient::new(client))
236244
}
237245
}
246+
247+
#[cfg(target_arch = "wasm32")]
248+
pub(crate) fn http_connector(
249+
custom: Option<Arc<dyn HttpConnector>>,
250+
) -> crate::Result<Arc<dyn HttpConnector>> {
251+
match custom {
252+
Some(x) => Ok(x),
253+
None => Err(crate::Error::NotSupported {
254+
source: "WASM32 architectures must provide an HTTPConnector"
255+
.to_string()
256+
.into(),
257+
}),
258+
}
259+
}
260+
261+
#[cfg(not(target_arch = "wasm32"))]
262+
pub(crate) fn http_connector(
263+
custom: Option<Arc<dyn HttpConnector>>,
264+
) -> crate::Result<Arc<dyn HttpConnector>> {
265+
match custom {
266+
Some(x) => Ok(x),
267+
None => Ok(Arc::new(ReqwestConnector {})),
268+
}
269+
}

‎src/client/mod.rs

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
2020
pub(crate) mod backoff;
2121

22+
#[cfg(not(target_arch = "wasm32"))]
2223
mod dns;
2324

2425
#[cfg(test)]
@@ -48,22 +49,25 @@ pub use body::{HttpRequest, HttpRequestBody, HttpResponse, HttpResponseBody};
4849
pub(crate) mod builder;
4950

5051
mod connection;
51-
pub use connection::{
52-
HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpService, ReqwestConnector,
53-
};
52+
pub(crate) use connection::http_connector;
53+
#[cfg(not(target_arch = "wasm32"))]
54+
pub use connection::ReqwestConnector;
55+
pub use connection::{HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpService};
5456

5557
#[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))]
5658
pub(crate) mod parts;
5759

5860
use async_trait::async_trait;
5961
use reqwest::header::{HeaderMap, HeaderValue};
60-
use reqwest::{Client, ClientBuilder, NoProxy, Proxy};
6162
use serde::{Deserialize, Serialize};
6263
use std::collections::HashMap;
6364
use std::str::FromStr;
6465
use std::sync::Arc;
6566
use std::time::Duration;
6667

68+
#[cfg(not(target_arch = "wasm32"))]
69+
use reqwest::{NoProxy, Proxy};
70+
6771
use crate::config::{fmt_duration, ConfigValue};
6872
use crate::path::Path;
6973
use crate::{GetOptions, Result};
@@ -195,8 +199,10 @@ impl FromStr for ClientConfigKey {
195199
/// This is used to configure the client to trust a specific certificate. See
196200
/// [Self::from_pem] for an example
197201
#[derive(Debug, Clone)]
202+
#[cfg(not(target_arch = "wasm32"))]
198203
pub struct Certificate(reqwest::tls::Certificate);
199204

205+
#[cfg(not(target_arch = "wasm32"))]
200206
impl Certificate {
201207
/// Create a `Certificate` from a PEM encoded certificate.
202208
///
@@ -243,6 +249,7 @@ impl Certificate {
243249
#[derive(Debug, Clone)]
244250
pub struct ClientOptions {
245251
user_agent: Option<ConfigValue<HeaderValue>>,
252+
#[cfg(not(target_arch = "wasm32"))]
246253
root_certificates: Vec<Certificate>,
247254
content_type_map: HashMap<String, String>,
248255
default_content_type: Option<String>,
@@ -276,6 +283,7 @@ impl Default for ClientOptions {
276283
// we opt for a slightly higher default timeout of 30 seconds
277284
Self {
278285
user_agent: None,
286+
#[cfg(not(target_arch = "wasm32"))]
279287
root_certificates: Default::default(),
280288
content_type_map: Default::default(),
281289
default_content_type: None,
@@ -402,6 +410,7 @@ impl ClientOptions {
402410
///
403411
/// This can be used to connect to a server that has a self-signed
404412
/// certificate for example.
413+
#[cfg(not(target_arch = "wasm32"))]
405414
pub fn with_root_certificate(mut self, certificate: Certificate) -> Self {
406415
self.root_certificates.push(certificate);
407416
self
@@ -614,8 +623,9 @@ impl ClientOptions {
614623
.with_connect_timeout(Duration::from_secs(1))
615624
}
616625

617-
pub(crate) fn client(&self) -> Result<Client> {
618-
let mut builder = ClientBuilder::new();
626+
#[cfg(not(target_arch = "wasm32"))]
627+
pub(crate) fn client(&self) -> Result<reqwest::Client> {
628+
let mut builder = reqwest::ClientBuilder::new();
619629

620630
match &self.user_agent {
621631
Some(user_agent) => builder = builder.user_agent(user_agent.get()?),
@@ -799,7 +809,7 @@ mod cloud {
799809
use crate::client::token::{TemporaryToken, TokenCache};
800810
use crate::RetryConfig;
801811

802-
/// A [`CredentialProvider`] that uses [`Client`] to fetch temporary tokens
812+
/// A [`CredentialProvider`] that uses [`HttpClient`] to fetch temporary tokens
803813
#[derive(Debug)]
804814
pub(crate) struct TokenCredentialProvider<T: TokenProvider> {
805815
inner: T,

‎src/gcp/builder.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider};
18+
use crate::client::{http_connector, HttpConnector, TokenCredentialProvider};
1919
use crate::gcp::client::{GoogleCloudStorageClient, GoogleCloudStorageConfig};
2020
use crate::gcp::credential::{
2121
ApplicationDefaultCredentials, InstanceCredentialProvider, ServiceAccountCredentials,
@@ -427,7 +427,9 @@ impl GoogleCloudStorageBuilder {
427427
self
428428
}
429429

430-
/// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`]
430+
/// The [`HttpConnector`] to use
431+
///
432+
/// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
431433
pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
432434
self.http_connector = Some(Arc::new(connector));
433435
self
@@ -442,9 +444,7 @@ impl GoogleCloudStorageBuilder {
442444

443445
let bucket_name = self.bucket_name.ok_or(Error::MissingBucketName {})?;
444446

445-
let http = self
446-
.http_connector
447-
.unwrap_or_else(|| Arc::new(ReqwestConnector::default()));
447+
let http = http_connector(self.http_connector)?;
448448

449449
// First try to initialize from the service account information.
450450
let service_account_credentials =

‎src/http/mod.rs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use url::Url;
4141

4242
use crate::client::get::GetClientExt;
4343
use crate::client::header::get_etag;
44-
use crate::client::{HttpConnector, ReqwestConnector};
44+
use crate::client::{http_connector, HttpConnector};
4545
use crate::http::client::Client;
4646
use crate::path::Path;
4747
use crate::{
@@ -237,7 +237,9 @@ impl HttpBuilder {
237237
self
238238
}
239239

240-
/// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`]
240+
/// The [`HttpConnector`] to use
241+
///
242+
/// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided
241243
pub fn with_http_connector<C: HttpConnector>(mut self, connector: C) -> Self {
242244
self.http_connector = Some(Arc::new(connector));
243245
self
@@ -248,10 +250,7 @@ impl HttpBuilder {
248250
let url = self.url.ok_or(Error::MissingUrl)?;
249251
let parsed = Url::parse(&url).map_err(|source| Error::UnableToParseUrl { url, source })?;
250252

251-
let client = match self.http_connector {
252-
None => ReqwestConnector::default().connect(&self.client_options)?,
253-
Some(x) => x.connect(&self.client_options)?,
254-
};
253+
let client = http_connector(self.http_connector)?.connect(&self.client_options)?;
255254

256255
Ok(HttpStore {
257256
client: Arc::new(Client::new(

‎src/lib.rs

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -497,12 +497,6 @@
497497
//! [`webpki-roots`]: https://crates.io/crates/webpki-roots
498498
//!
499499
500-
#[cfg(all(
501-
target_arch = "wasm32",
502-
any(feature = "gcp", feature = "aws", feature = "azure", feature = "http")
503-
))]
504-
compile_error!("Features 'gcp', 'aws', 'azure', 'http' are not supported on wasm.");
505-
506500
#[cfg(feature = "aws")]
507501
pub mod aws;
508502
#[cfg(feature = "azure")]
@@ -530,10 +524,13 @@ pub mod client;
530524

531525
#[cfg(feature = "cloud")]
532526
pub use client::{
533-
backoff::BackoffConfig, retry::RetryConfig, Certificate, ClientConfigKey, ClientOptions,
534-
CredentialProvider, StaticCredentialProvider,
527+
backoff::BackoffConfig, retry::RetryConfig, ClientConfigKey, ClientOptions, CredentialProvider,
528+
StaticCredentialProvider,
535529
};
536530

531+
#[cfg(all(feature = "cloud", not(target_arch = "wasm32")))]
532+
pub use client::Certificate;
533+
537534
#[cfg(feature = "cloud")]
538535
mod config;
539536

@@ -1083,8 +1080,6 @@ impl GetResult {
10831080
.await
10841081
}
10851082
GetResultPayload::Stream(s) => collect_bytes(s, Some(len)).await,
1086-
#[cfg(target_arch = "wasm32")]
1087-
_ => unimplemented!("File IO not implemented on wasm32."),
10881083
}
10891084
}
10901085

@@ -1110,8 +1105,6 @@ impl GetResult {
11101105
local::chunked_stream(file, path, self.range, CHUNK_SIZE)
11111106
}
11121107
GetResultPayload::Stream(s) => s,
1113-
#[cfg(target_arch = "wasm32")]
1114-
_ => unimplemented!("File IO not implemented on wasm32."),
11151108
}
11161109
}
11171110
}

‎src/parse.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,13 @@ where
201201
let url = &url[..url::Position::BeforePath];
202202
builder_opts!(crate::http::HttpBuilder, url, _options)
203203
}
204-
#[cfg(not(all(feature = "aws", feature = "azure", feature = "gcp", feature = "http")))]
204+
#[cfg(not(all(
205+
feature = "aws",
206+
feature = "azure",
207+
feature = "gcp",
208+
feature = "http",
209+
not(target_arch = "wasm32")
210+
)))]
205211
s => {
206212
return Err(super::Error::Generic {
207213
store: "parse_url",

0 commit comments

Comments
 (0)