diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index d245a887a63..68cd3b56ee5 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -499,3 +499,7 @@ simulatable URIs [Aa]bliterated [Uu]npatched +Alibaba +OSS +AccessKey +SigV4 diff --git a/nativelink-config/examples/stores-config.json5 b/nativelink-config/examples/stores-config.json5 index 0748ce406b2..de0f3053b37 100644 --- a/nativelink-config/examples/stores-config.json5 +++ b/nativelink-config/examples/stores-config.json5 @@ -117,6 +117,18 @@ }, { name: "8", + "experimental_cloud_object_store": { + "provider": "oss", + "region": "cn-hangzhou", + "bucket": "example-build-cache", + "access_key_id": "oss_access_key_id", + "secret_access_key": "oss_secret_access_key", + "cname": false, + "key_prefix": "cas/" + } + }, + { + name: "9", "ontap_s3_existence_cache": { "index_path": "/path/to/cache/index.json", "sync_interval_seconds": 300, @@ -129,7 +141,7 @@ } }, { - name: "9", + name: "10", "verify": { "backend": { "memory": { @@ -143,7 +155,7 @@ } }, { - name: "10", + name: "11", "completeness_checking": { "backend": { "filesystem": { @@ -162,7 +174,7 @@ } }, { - name: "11", + name: "12", "compression": { "compression_algorithm": { "lz4": {} @@ -179,7 +191,7 @@ } }, { - name: "12", + name: "13", "dedup": { "index_store": { "memory": { @@ -218,7 +230,7 @@ } }, { - name: "13", + name: "14", "existence_cache": { "backend": { "memory": { @@ -234,7 +246,7 @@ } }, { - name: "14", + name: "15", "fast_slow": { "fast": { "filesystem": { @@ -257,7 +269,7 @@ } }, { - name: "15", + name: "16", "shard": { "stores": [ { @@ -273,7 +285,7 @@ } }, { - name: "16", + name: "17", "filesystem": { "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", @@ -283,13 +295,13 @@ } }, { - name: "17", + name: "18", "ref_store": { "name": "FS_CONTENT_STORE" } }, { - name: "18", + name: "19", "size_partitioning": { "size": "128mib", "lower_store": { @@ -306,7 +318,7 @@ } }, { - name: "19", + name: "20", "grpc": { "instance_name": "main", "endpoints": [ @@ -327,7 +339,7 @@ } }, { - name: "20", + name: "21", "redis_store": { "addresses": [ "redis://127.0.0.1:6379/", @@ -336,11 +348,11 @@ } }, { - name: "21", + name: "22", "noop": {} }, { - name: "22", + name: "23", "experimental_mongo": { "connection_string": "mongodb://localhost:27017", "database": "nativelink", diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index 6d65aa49493..1fb3945df85 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -230,6 +230,28 @@ pub enum StoreSpec { /// } /// } /// ``` + /// + /// 7. **Alibaba Cloud OSS:** + /// Uses the OSS S3-compatible endpoint and AWS `SigV4`. Supply the OSS + /// access key explicitly per store, for example by shell-expanding + /// `OSS_ACCESS_KEY_ID` and `OSS_ACCESS_KEY_SECRET` in the config. + /// Set `cname` to true with an explicit HTTPS `endpoint` to use a custom + /// domain bound to the bucket, without adding the bucket to the host or + /// path. The default is false. Configure the domain binding, DNS, and + /// TLS certificate before starting the store. + /// + /// **Example JSON Config:** + /// ```json + /// "experimental_cloud_object_store": { + /// "provider": "oss", + /// "region": "cn-hangzhou", + /// "bucket": "example-build-cache", + /// "access_key_id": "oss_access_key_id", + /// "secret_access_key": "oss_secret_access_key", + /// "cname": false, + /// "key_prefix": "cas/" + /// } + /// ``` ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec), /// ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store @@ -948,6 +970,84 @@ pub struct ExperimentalOciSpec { pub common: CommonObjectSpec, } +/// Alibaba Cloud OSS through its S3-compatible API. +#[derive(Serialize, Deserialize, Default, Clone)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct ExperimentalOssSpec { + /// OSS region, for example `cn-hangzhou`. Used for AWS `SigV4` signing + /// and to derive the default S3-compatible endpoint. + #[serde(deserialize_with = "convert_string_with_shellexpand")] + pub region: String, + + /// Bucket name to use as the backend. + #[serde(deserialize_with = "convert_string_with_shellexpand")] + pub bucket: String, + + /// Access key ID for this store. Required; no AWS credential-chain fallback. + /// Use shell expansion, for example `${OSS_ACCESS_KEY_ID}`. + #[serde(deserialize_with = "convert_string_with_shellexpand")] + pub access_key_id: String, + + /// Access key secret for this store. Use `${OSS_ACCESS_KEY_SECRET}` to + /// keep it out of the config file. Independent of AWS S3 store credentials. + #[serde(deserialize_with = "convert_string_with_shellexpand")] + pub secret_access_key: String, + + /// Optional STS token for this store, for example `${OSS_SESSION_TOKEN}`. + #[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")] + pub session_token: Option, + + /// Optional HTTPS S3-compatible service endpoint, without a bucket or path. + /// Default: `https://s3.oss-{region}.aliyuncs.com`. + /// With `cname: true`, this must be the custom domain bound to the bucket. + /// Native OSS REST endpoints are not interchangeable with this endpoint. + #[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")] + pub endpoint: Option, + + /// Use `endpoint` as the bucket's custom domain without adding the bucket + /// to the host or path. Requires an explicit HTTPS endpoint, an OSS domain + /// binding, and a valid certificate for that domain. Default: false. + #[serde(default)] + pub cname: bool, + + /// Prefix prepended verbatim to each object key, for example `cas/`. + #[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")] + pub key_prefix: Option, + + /// Retry policy for the shared S3 store. SDK retries are disabled to avoid + /// multiplying attempts. Default: the standard store retry policy. + #[serde(default)] + pub retry: Retry, + + /// Maximum concurrent part requests per upload. Must be positive. + /// Default: 2. Queued parts are bounded by the same count; this is not + /// a global byte budget. Part size is chosen by the shared S3 store. + #[serde( + default, + deserialize_with = "convert_optional_numeric_with_shellexpand" + )] + pub multipart_max_concurrent_uploads: Option, +} + +impl core::fmt::Debug for ExperimentalOssSpec { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ExperimentalOssSpec") + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("cname", &self.cname) + .field("key_prefix", &self.key_prefix) + .field("retry", &self.retry) + .field( + "multipart_max_concurrent_uploads", + &self.multipart_max_concurrent_uploads, + ) + .field("credentials", &"") + .finish() + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] @@ -1266,6 +1366,7 @@ pub enum ExperimentalCloudObjectSpec { Ontap(ExperimentalOntapS3Spec), R2(ExperimentalR2Spec), Oci(ExperimentalOciSpec), + Oss(ExperimentalOssSpec), } impl Default for ExperimentalCloudObjectSpec { diff --git a/nativelink-config/tests/deserialization_test.rs b/nativelink-config/tests/deserialization_test.rs index cf882ffd622..4f04c734964 100644 --- a/nativelink-config/tests/deserialization_test.rs +++ b/nativelink-config/tests/deserialization_test.rs @@ -796,3 +796,68 @@ fn http_listener_tcp_nodelay_defaults_and_overrides() { assert!(serde_json5::from_str::(&input).is_err()); } } + +#[test] +fn oss_provider_accepts_minimal_config_and_rejects_unrelated_options() { + use nativelink_config::stores::{ExperimentalCloudObjectSpec, StoreSpec}; + let config = r#"{"experimental_cloud_object_store":{"provider":"oss","region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"OSS_TEST_KEY","secret_access_key":"OSS_TEST_SECRET"}}"#; + let parsed: StoreSpec = serde_json5::from_str(config).unwrap(); + assert!(matches!( + parsed, + StoreSpec::ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec::Oss(_)) + )); + for field in [ + "consider_expired_after_s", + "max_retry_buffer_per_request", + "max_concurrent_uploads", + "upload_queue_timeout_s", + ] { + let config = format!( + r#"{{"provider":"oss","region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"OSS_TEST_KEY","secret_access_key":"OSS_TEST_SECRET","{field}":1}}"# + ); + assert!( + serde_json5::from_str::(&config).is_err(), + "unexpected OSS field accepted: {field}" + ); + } +} + +#[test] +fn oss_credentials_are_required_and_redacted_from_debug() { + use nativelink_config::stores::ExperimentalOssSpec; + let missing = r#"{"region":"cn-hangzhou","bucket":"test-bucket"}"#; + assert!(serde_json5::from_str::(missing).is_err()); + let spec = ExperimentalOssSpec { + access_key_id: "PRIVATE_OSS_KEY_ID".into(), + secret_access_key: "PRIVATE_OSS_SECRET".into(), + session_token: Some("PRIVATE_OSS_TOKEN".into()), + ..Default::default() + }; + let debug = format!("{spec:?}"); + for credential in [ + "PRIVATE_OSS_KEY_ID", + "PRIVATE_OSS_SECRET", + "PRIVATE_OSS_TOKEN", + ] { + assert!(!debug.contains(credential)); + } +} + +#[test] +fn oss_cname_defaults_to_false_and_round_trips() { + use nativelink_config::stores::ExperimentalOssSpec; + + let config = r#"{"region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"TESTKEY","secret_access_key":"TESTSECRET"}"#; + let mut spec: ExperimentalOssSpec = serde_json5::from_str(config).unwrap(); + assert!(!spec.cname); + spec.cname = true; + spec.endpoint = Some("https://cache.example.com".into()); + let round_trip: ExperimentalOssSpec = + serde_json::from_value(serde_json::to_value(&spec).unwrap()).unwrap(); + assert!(round_trip.cname); + assert_eq!(round_trip.endpoint, spec.endpoint); + for invalid in ["null", "1", "\"true\""] { + let input = format!("{},\"cname\":{invalid}}}", &config[..config.len() - 1]); + assert!(serde_json5::from_str::(&input).is_err()); + } +} diff --git a/nativelink-store/BUILD.bazel b/nativelink-store/BUILD.bazel index 90ecfcf2caf..dd815815d55 100644 --- a/nativelink-store/BUILD.bazel +++ b/nativelink-store/BUILD.bazel @@ -51,6 +51,7 @@ rust_library( "src/oci_store.rs", "src/ontap_s3_existence_cache_store.rs", "src/ontap_s3_store.rs", + "src/oss_store.rs", "src/r2_store.rs", "src/redis_store.rs", "src/redis_utils/aggregate_types.rs", @@ -163,6 +164,7 @@ rust_test_suite( "tests/oci_store_test.rs", "tests/ontap_s3_existence_cache_store_test.rs", "tests/ontap_s3_store_test.rs", + "tests/oss_store_test.rs", "tests/r2_store_test.rs", "tests/redis_store_test.rs", "tests/ref_store_test.rs", diff --git a/nativelink-store/src/default_store_factory.rs b/nativelink-store/src/default_store_factory.rs index 36de09fc429..3a63ab84a8b 100644 --- a/nativelink-store/src/default_store_factory.rs +++ b/nativelink-store/src/default_store_factory.rs @@ -39,6 +39,7 @@ use crate::noop_store::NoopStore; use crate::oci_store::OciStore; use crate::ontap_s3_existence_cache_store::OntapS3ExistenceCache; use crate::ontap_s3_store::OntapS3Store; +use crate::oss_store::OssStore; use crate::r2_store::R2Store; use crate::redis_store::RedisStore; use crate::ref_store::RefStore; @@ -78,6 +79,9 @@ pub fn store_factory<'a>( ExperimentalCloudObjectSpec::R2(r2_config) => { R2Store::new(r2_config, SystemTime::now).await? } + ExperimentalCloudObjectSpec::Oss(oss_config) => { + OssStore::new(oss_config, SystemTime::now)? + } ExperimentalCloudObjectSpec::Oci(oci_config) => { OciStore::new(oci_config, SystemTime::now).await? } diff --git a/nativelink-store/src/lib.rs b/nativelink-store/src/lib.rs index 88a524a281b..471f1b04f09 100644 --- a/nativelink-store/src/lib.rs +++ b/nativelink-store/src/lib.rs @@ -34,6 +34,7 @@ pub mod noop_store; pub mod oci_store; pub mod ontap_s3_existence_cache_store; pub mod ontap_s3_store; +pub mod oss_store; pub mod r2_store; pub mod redis_store; mod redis_utils; diff --git a/nativelink-store/src/oss_store.rs b/nativelink-store/src/oss_store.rs new file mode 100644 index 00000000000..e67ef91dc74 --- /dev/null +++ b/nativelink-store/src/oss_store.rs @@ -0,0 +1,199 @@ +// Copyright 2026 Trace Machina, Inc. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::time::Duration; +use std::sync::Arc; + +use aws_config::{AppName, BehaviorVersion}; +use aws_sdk_s3::Client; +use aws_sdk_s3::config::endpoint::{Endpoint, EndpointFuture, Params, ResolveEndpoint}; +use aws_sdk_s3::config::retry::RetryConfig; +use aws_sdk_s3::config::{ + Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation, +}; +use aws_smithy_runtime_api::client::http::HttpClient as SmithyHttpClient; +use aws_smithy_types::endpoint::EndpointAuthScheme; +use nativelink_config::stores::{CommonObjectSpec, ExperimentalAwsSpec, ExperimentalOssSpec}; +use nativelink_error::{Code, Error, make_err}; +use nativelink_util::instant_wrapper::InstantWrapper; + +use crate::common_s3_utils::TlsClient; +use crate::s3_store::S3Store; + +/// OSS S3-compatible endpoint adapter. Transfer logic stays in `S3Store`. +/// See . +#[derive(Debug, Clone, Copy)] +pub struct OssStore; + +#[derive(Debug)] +struct CnameResolver(Endpoint); + +impl ResolveEndpoint for CnameResolver { + fn resolve_endpoint(&self, _params: &Params) -> EndpointFuture<'_> { + EndpointFuture::ready(Ok(self.0.clone())) + } +} + +impl OssStore { + #[allow(clippy::new_ret_no_self)] + pub fn new( + spec: &ExperimentalOssSpec, + now_fn: NowFn, + ) -> Result>, Error> + where + I: InstantWrapper, + NowFn: Fn() -> I + Send + Sync + Unpin + 'static, + { + let aws_spec = Self::build_aws_spec(spec)?; + let http_client = TlsClient::new(&aws_spec.common)?; + Self::new_with_http_client(spec, http_client, now_fn) + } + + /// Allows wire tests to supply an HTTP client without contacting OSS. + #[allow(clippy::new_ret_no_self)] + pub fn new_with_http_client( + spec: &ExperimentalOssSpec, + http_client: C, + now_fn: NowFn, + ) -> Result>, Error> + where + I: InstantWrapper, + NowFn: Fn() -> I + Send + Sync + Unpin + 'static, + C: SmithyHttpClient + Clone + 'static, + { + let aws_spec = Self::build_aws_spec(spec)?; + let region = Region::new(spec.region.clone()); + let endpoint = Self::endpoint(spec)?; + let mut config = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .app_name(AppName::new("nativelink").expect("valid app name")) + .region(region) + // OSS documents virtual-hosted addressing for its S3 interface. + .force_path_style(false) + // Do not add optional trailing checksums / aws-chunked encoding. + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .timeout_config( + aws_config::timeout::TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(15)) + .operation_attempt_timeout(Duration::from_mins(2)) + .build(), + ) + .credentials_provider(Credentials::new( + &spec.access_key_id, + &spec.secret_access_key, + spec.session_token.clone(), + None, + "oss-config", + )) + .http_client(http_client); + if spec.cname { + config = config.endpoint_resolver(CnameResolver( + Endpoint::builder() + .url(endpoint) + .auth_scheme( + EndpointAuthScheme::with_capacity("sigv4", 3) + .put("disableDoubleEncoding", true) + .put("signingName", "s3") + .put("signingRegion", spec.region.clone()), + ) + .build(), + )); + } else { + config = config.endpoint_url(endpoint); + } + S3Store::new_with_client_and_jitter( + &aws_spec, + Client::from_conf(config.build()), + spec.retry.make_jitter_fn(), + now_fn, + ) + } + + pub fn endpoint(spec: &ExperimentalOssSpec) -> Result { + if spec.cname && spec.endpoint.is_none() { + return Err(make_err!( + Code::InvalidArgument, + "OSS cname requires an explicit endpoint bound to the bucket" + )); + } + let endpoint = spec + .endpoint + .clone() + .unwrap_or_else(|| format!("https://s3.oss-{}.aliyuncs.com", spec.region)); + let url = url::Url::parse(&endpoint) + .map_err(|_| make_err!(Code::InvalidArgument, "Invalid OSS endpoint"))?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + { + return Err(make_err!( + Code::InvalidArgument, + "OSS endpoint must be an HTTPS URL without credentials, path, query or fragment" + )); + } + Ok(endpoint) + } + + pub fn build_aws_spec(spec: &ExperimentalOssSpec) -> Result { + if spec.region.is_empty() + || !spec + .region + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + { + return Err(make_err!(Code::InvalidArgument, "Invalid OSS region")); + } + let bucket = spec.bucket.as_bytes(); + if !(3..=63).contains(&bucket.len()) + || !bucket[0].is_ascii_alphanumeric() + || !bucket[bucket.len() - 1].is_ascii_alphanumeric() + || !bucket + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-') + { + return Err(make_err!(Code::InvalidArgument, "Invalid OSS bucket name")); + } + if spec.access_key_id.trim().is_empty() || spec.secret_access_key.trim().is_empty() { + return Err(make_err!( + Code::InvalidArgument, + "OSS requires explicit access_key_id and secret_access_key" + )); + } + let concurrency = spec.multipart_max_concurrent_uploads.unwrap_or(2); + if concurrency == 0 || concurrency > tokio::sync::Semaphore::MAX_PERMITS { + return Err(make_err!( + Code::InvalidArgument, + "Invalid OSS multipart concurrency" + )); + } + Self::endpoint(spec)?; + Ok(ExperimentalAwsSpec { + region: spec.region.clone(), + bucket: spec.bucket.clone(), + common: CommonObjectSpec { + key_prefix: spec.key_prefix.clone(), + retry: spec.retry.clone(), + multipart_max_concurrent_uploads: Some(concurrency), + ..Default::default() + }, + ..Default::default() + }) + } +} diff --git a/nativelink-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs index 33d116b854b..22bb6537d2e 100644 --- a/nativelink-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -480,8 +480,8 @@ where } let upload_parts = move || async move { - // This will ensure we only have `multipart_max_concurrent_uploads` * `bytes_per_upload_part` - // bytes in memory at any given time waiting to be uploaded. + // Bound both queued parts and in-flight requests so a fast source + // cannot buffer an entire object behind a slow backend. let (tx, mut rx) = mpsc::channel(self.multipart_max_concurrent_uploads); let read_stream_fut = async move { @@ -563,7 +563,7 @@ where total_uploaded = result?; }, // Return error or wait for other futures. Some(upload_result) = upload_futures.next() => completed_parts.push(upload_result?), - Some(fut) = rx.recv() => upload_futures.push(fut), + Some(fut) = rx.recv(), if upload_futures.len() < self.multipart_max_concurrent_uploads => upload_futures.push(fut), } } @@ -633,7 +633,7 @@ where offset: u64, length: Option, ) -> Result<(), Error> { - if is_zero_digest(key.borrow()) { + if is_zero_digest(key.borrow()) || length == Some(0) { writer .send_eof() .err_tip(|| "Failed to send zero EOF in filesystem store get_part")?; @@ -642,7 +642,7 @@ where let s3_path = &self.make_s3_path(&key); let end_read_byte = length - .map_or(Some(None), |length| Some(offset.checked_add(length))) + .map_or(Some(None), |length| Some(offset.checked_add(length - 1))) .err_tip(|| "Integer overflow protection triggered")?; self.retrier diff --git a/nativelink-store/tests/oci_store_test.rs b/nativelink-store/tests/oci_store_test.rs index c3e5ca75765..8cb24f19707 100644 --- a/nativelink-store/tests/oci_store_test.rs +++ b/nativelink-store/tests/oci_store_test.rs @@ -250,7 +250,7 @@ async fn ranged_get_sends_range_header_path_style() -> Result<(), Error> { .uri(format!( "https://{HOST}/{BUCKET}/{VALID_HASH1}-1000?x-id=GetObject" )) - .header("range", format!("bytes={OFFSET}-{}", OFFSET + LENGTH)) + .header("range", format!("bytes={OFFSET}-{}", OFFSET + LENGTH - 1)) .body(SdkBody::empty()) .unwrap(), http::Response::builder() diff --git a/nativelink-store/tests/oss_store_test.rs b/nativelink-store/tests/oss_store_test.rs new file mode 100644 index 00000000000..e7c1b0d25da --- /dev/null +++ b/nativelink-store/tests/oss_store_test.rs @@ -0,0 +1,767 @@ +// Copyright 2026 Trace Machina, Inc. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! OSS-specific config and S3 wire behavior. No real bucket or credentials required. + +use std::sync::Arc; + +use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient}; +use aws_smithy_types::body::SdkBody; +use hmac::{Hmac, Mac}; +use http::StatusCode; +use nativelink_config::stores::{ExperimentalOssSpec, Retry}; +use nativelink_error::{Code, Error}; +use nativelink_macro::nativelink_test; +use nativelink_store::oss_store::OssStore; +use nativelink_store::s3_store::S3Store; +use nativelink_util::buf_channel::make_buf_channel_pair; +use nativelink_util::instant_wrapper::MockInstantWrapped; +use nativelink_util::spawn; +use nativelink_util::store_trait::{StoreKey, StoreLike, UploadSizeInfo}; +use sha2::{Digest, Sha256}; + +const HOST: &str = "test-bucket.s3.oss-cn-hangzhou.aliyuncs.com"; +const NOW: fn() -> MockInstantWrapped = MockInstantWrapped::default; + +type TestStore = S3Store MockInstantWrapped>; + +fn spec() -> ExperimentalOssSpec { + ExperimentalOssSpec { + region: "cn-hangzhou".into(), + bucket: "test-bucket".into(), + access_key_id: "TESTKEY".into(), + secret_access_key: "TESTSECRET".into(), + key_prefix: Some("cas/".into()), + retry: Retry { + max_retries: 1, + delay: 0.0, + jitter: 0.0, + ..Default::default() + }, + ..Default::default() + } +} + +fn store(mock: StaticReplayClient) -> Result, Error> { + OssStore::new_with_http_client(&spec(), mock, NOW) +} + +fn cname_spec() -> ExperimentalOssSpec { + ExperimentalOssSpec { + cname: true, + endpoint: Some("https://cache.example.com".into()), + session_token: Some("TESTTOKEN".into()), + ..spec() + } +} + +// Independently verify the emitted signature, including the custom host and +// singly encoded S3 path, rather than just checking the credential scope. +fn assert_cname_signature(request: &aws_smithy_runtime_api::client::orchestrator::HttpRequest) { + let uri = url::Url::parse(request.uri()).unwrap(); + assert_eq!(uri.host_str(), Some("cache.example.com")); + let headers = request.headers(); + let authorization = headers.get("authorization").unwrap(); + let signed_headers = authorization + .split("SignedHeaders=") + .nth(1) + .unwrap() + .split(',') + .next() + .unwrap(); + let mut canonical_headers = String::new(); + for name in signed_headers.split(';') { + let value = if name == "host" { + "cache.example.com" + } else { + headers.get(name).unwrap() + }; + canonical_headers.push_str(name); + canonical_headers.push(':'); + canonical_headers.push_str(value.trim()); + canonical_headers.push('\n'); + } + assert!(signed_headers.split(';').any(|name| name == "host")); + let mut query: Vec<_> = uri + .query() + .into_iter() + .flat_map(|query| query.split('&')) + .map(|pair| { + if pair.contains('=') { + pair.to_string() + } else { + format!("{pair}=") + } + }) + .collect(); + query.sort_unstable(); + let canonical_request = format!( + "{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{}", + request.method(), + uri.path(), + query.join("&"), + headers.get("x-amz-content-sha256").unwrap(), + ); + let date = headers.get("x-amz-date").unwrap(); + let scope = format!("{}/cn-hangzhou/s3/aws4_request", &date[..8]); + assert!(authorization.contains(&format!("Credential=TESTKEY/{scope},"))); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{date}\n{scope}\n{:x}", + Sha256::digest(canonical_request.as_bytes()) + ); + let mut key = b"AWS4TESTSECRET".to_vec(); + for value in [&date[..8], "cn-hangzhou", "s3", "aws4_request"] { + let mut mac = Hmac::::new_from_slice(&key).unwrap(); + mac.update(value.as_bytes()); + key = mac.finalize().into_bytes().to_vec(); + } + let mut mac = Hmac::::new_from_slice(&key).unwrap(); + mac.update(string_to_sign.as_bytes()); + assert!(authorization.ends_with(&format!("Signature={:x}", mac.finalize().into_bytes()))); + assert_eq!(headers.get("x-amz-security-token"), Some("TESTTOKEN")); +} + +#[test] +fn cname_requires_an_explicit_valid_endpoint() { + assert_eq!( + OssStore::build_aws_spec(&ExperimentalOssSpec { + cname: true, + ..spec() + }) + .unwrap_err() + .code, + Code::InvalidArgument + ); + for endpoint in [ + "http://cache.example.com", + "https://user:hidden@cache.example.com", + "https://cache.example.com/bucket", + "https://cache.example.com/?token=hidden", + "https://cache.example.com/#hidden", + "", + ] { + let error = OssStore::build_aws_spec(&ExperimentalOssSpec { + endpoint: Some(endpoint.into()), + ..cname_spec() + }) + .unwrap_err(); + assert_eq!(error.code, Code::InvalidArgument); + assert!(!error.to_string().contains("hidden")); + } +} + +#[nativelink_test] +async fn cname_reads_and_writes_use_bound_host_and_valid_signatures() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .header("content-length", "5") + .body(SdkBody::empty()) + .unwrap(), + ), + event(StatusCode::PARTIAL_CONTENT, "ell"), + event(StatusCode::OK, ""), + ]); + let store = OssStore::new_with_http_client(&cname_spec(), mock.clone(), NOW)?; + let key = StoreKey::new_str("a b%?c"); + assert_eq!(store.has(key.clone()).await?, Some(5)); + assert_eq!( + store.get_part_unchunked(key.clone(), 1, Some(3)).await?, + "ell" + ); + store.update_oneshot(key, "hello".into()).await?; + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 3); + for request in &requests { + assert_eq!( + url::Url::parse(request.uri()).unwrap().path(), + "/cas/a%20b%25%3Fc" + ); + assert_cname_signature(request); + } + assert_eq!(requests[0].method(), "HEAD"); + assert_eq!(requests[1].method(), "GET"); + assert_eq!(requests[1].headers().get("range"), Some("bytes=1-3")); + assert_eq!(requests[2].method(), "PUT"); + assert_ne!( + requests[2].headers().get("content-encoding"), + Some("aws-chunked") + ); + assert_eq!(requests[2].headers().get("content-length"), Some("5")); + Ok(()) +} + +#[nativelink_test] +async fn cname_multipart_completion_and_abort_keep_bound_host() -> Result<(), Error> { + for max_size in [10, 2] { + let mock = StaticReplayClient::new(vec![ + event( + StatusCode::OK, + "upload-1", + ), + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .header("etag", "part-etag") + .body(SdkBody::empty()) + .unwrap(), + ), + event( + StatusCode::OK, + "object-etag", + ), + ]); + let store = OssStore::new_with_http_client(&cname_spec(), mock.clone(), NOW)?; + let (mut tx, rx) = make_buf_channel_pair(); + let upload = store.update( + StoreKey::new_str("key"), + rx, + UploadSizeInfo::MaxSize(max_size), + ); + let send = async { + tx.send("hello".into()).await?; + tx.send_eof() + }; + let (written, sent) = tokio::join!(upload, send); + sent?; + let final_method = if max_size == 10 { + assert_eq!(written?, 5); + "POST" + } else { + assert_eq!(written.unwrap_err().code, Code::InvalidArgument); + "DELETE" + }; + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method(), "POST"); + assert!(requests[0].uri().contains("uploads")); + assert_eq!(requests[1].method(), "PUT"); + assert!(requests[1].uri().contains("partNumber=1")); + assert!(requests[1].uri().contains("uploadId=upload-1")); + assert_eq!(requests[2].method(), final_method); + assert!(requests[2].uri().contains("uploadId=upload-1")); + for request in &requests { + assert_eq!(url::Url::parse(request.uri()).unwrap().path(), "/cas/key"); + assert_cname_signature(request); + } + } + Ok(()) +} + +fn event(status: StatusCode, body: &'static str) -> ReplayEvent { + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .status(status) + .body(SdkBody::from(body)) + .unwrap(), + ) +} + +#[test] +fn endpoint_and_minimal_settings() -> Result<(), Error> { + assert_eq!( + OssStore::endpoint(&spec())?, + "https://s3.oss-cn-hangzhou.aliyuncs.com" + ); + let aws = OssStore::build_aws_spec(&spec())?; + assert_eq!(aws.region, "cn-hangzhou"); + assert_eq!(aws.bucket, "test-bucket"); + assert_eq!(aws.common.key_prefix.as_deref(), Some("cas/")); + assert_eq!(aws.common.retry.max_retries, 1); + assert_eq!(aws.common.consider_expired_after_s, 0); + assert_eq!(aws.common.multipart_max_concurrent_uploads, Some(2)); + Ok(()) +} + +#[test] +fn rejects_invalid_config_without_echoing_endpoint_credentials() { + for endpoint in [ + "http://example.com", + "https://user:secret@example.com", + "https://example.com/bucket", + "https://example.com/?secret=hidden", + "https://example.com/#fragment", + "not a url", + ] { + let err = OssStore::build_aws_spec(&ExperimentalOssSpec { + endpoint: Some(endpoint.into()), + ..spec() + }) + .unwrap_err(); + assert_eq!(err.code, Code::InvalidArgument); + assert!(!err.to_string().contains("hidden")); + assert!(!err.to_string().contains("secret@example")); + } + assert_eq!( + OssStore::build_aws_spec(&ExperimentalOssSpec { + multipart_max_concurrent_uploads: Some(0), + ..spec() + }) + .unwrap_err() + .code, + Code::InvalidArgument + ); + for bucket in [ + "", + "ab", + "Uppercase", + "with.dot", + "with/slash", + "-leading", + "trailing-", + ] { + assert_eq!( + OssStore::build_aws_spec(&ExperimentalOssSpec { + bucket: bucket.into(), + ..spec() + }) + .unwrap_err() + .code, + Code::InvalidArgument + ); + } + for region in ["", "../region", "region/", "region?token=hidden"] { + assert_eq!( + OssStore::build_aws_spec(&ExperimentalOssSpec { + region: region.into(), + ..spec() + }) + .unwrap_err() + .code, + Code::InvalidArgument + ); + } +} + +#[nativelink_test] +async fn head_uses_virtual_host_prefix_and_signing_region() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .status(StatusCode::OK) + .header("content-length", "17") + .body(SdkBody::empty()) + .unwrap(), + )]); + assert_eq!( + store(mock.clone())?.has(StoreKey::new_str("key")).await?, + Some(17) + ); + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method(), "HEAD"); + assert_eq!(requests[0].uri(), format!("https://{HOST}/cas/key")); + assert!( + requests[0] + .headers() + .get("authorization") + .unwrap() + .contains("/cn-hangzhou/s3/aws4_request") + ); + Ok(()) +} + +#[nativelink_test] +async fn head_missing_is_a_cache_miss() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event(StatusCode::NOT_FOUND, "")]); + assert_eq!(store(mock)?.has(StoreKey::new_str("missing")).await?, None); + Ok(()) +} + +#[nativelink_test] +async fn get_missing_reports_not_found() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event( + StatusCode::NOT_FOUND, + "NoSuchKey", + )]); + let err = store(mock)? + .get_part_unchunked(StoreKey::new_str("missing"), 0, None) + .await + .unwrap_err(); + assert_eq!(err.code, Code::NotFound); + Ok(()) +} + +#[nativelink_test] +async fn ranged_get_and_encoded_key_use_oss_endpoint() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event(StatusCode::PARTIAL_CONTENT, "bcd")]); + let bytes = store(mock.clone())? + .get_part_unchunked(StoreKey::new_str("a b?c"), 1, Some(3)) + .await?; + assert_eq!(bytes, "bcd"); + let requests: Vec<_> = mock.actual_requests().collect(); + assert!( + requests[0] + .uri() + .starts_with(&format!("https://{HOST}/cas/a%20b%3Fc")) + ); + assert_eq!(requests[0].headers().get("range"), Some("bytes=1-3")); + Ok(()) +} + +#[nativelink_test] +async fn single_put_does_not_enable_aws_chunked_encoding() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event(StatusCode::OK, "")]); + store(mock.clone())? + .update_oneshot(StoreKey::new_str("key"), "hello".into()) + .await?; + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + assert_eq!(request.method(), "PUT"); + assert_eq!(request.headers().get("content-length"), Some("5")); + assert!(request.headers().get("transfer-encoding").is_none()); + assert!( + request + .uri() + .starts_with(&format!("https://{HOST}/cas/key")) + ); + assert_ne!( + request.headers().get("content-encoding"), + Some("aws-chunked") + ); + assert!( + !request + .headers() + .get("x-amz-content-sha256") + .unwrap_or_default() + .contains("STREAMING") + ); + assert!( + request + .headers() + .get("x-amz-sdk-checksum-algorithm") + .is_none() + ); + Ok(()) +} + +#[nativelink_test] +async fn multipart_upload_initiates_uploads_and_completes() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ + event( + StatusCode::OK, + "upload-1", + ), + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .status(StatusCode::OK) + .header("etag", "part-etag") + .body(SdkBody::empty()) + .unwrap(), + ), + event( + StatusCode::OK, + "object-etag", + ), + ]); + let store = store(mock.clone())?; + let (mut tx, rx) = make_buf_channel_pair(); + let upload = store.update(StoreKey::new_str("key"), rx, UploadSizeInfo::MaxSize(10)); + let send = async { + tx.send("hello".into()).await?; + tx.send_eof() + }; + let (written, ()) = futures::try_join!(upload, send)?; + assert_eq!(written, 5); + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method(), "POST"); + assert!(requests[0].uri().contains("uploads")); + assert_eq!(requests[1].method(), "PUT"); + assert!(requests[1].uri().contains("partNumber=1")); + assert!(requests[1].uri().contains("uploadId=upload-1")); + assert_eq!(requests[2].method(), "POST"); + assert!(requests[2].uri().contains("uploadId=upload-1")); + let completion = std::str::from_utf8(requests[2].body().bytes().unwrap()).unwrap(); + assert!(completion.contains("1")); + assert!(completion.contains("part-etag")); + Ok(()) +} + +#[nativelink_test] +async fn multipart_size_error_aborts_without_publishing() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ + event( + StatusCode::OK, + "upload-1", + ), + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .status(StatusCode::OK) + .header("etag", "part-etag") + .body(SdkBody::empty()) + .unwrap(), + ), + event(StatusCode::NO_CONTENT, ""), + ]); + let store = store(mock.clone())?; + let (mut tx, rx) = make_buf_channel_pair(); + let upload = store.update(StoreKey::new_str("key"), rx, UploadSizeInfo::MaxSize(2)); + let send = async { + tx.send("hello".into()).await?; + tx.send_eof() + }; + let (written, sent) = tokio::join!(upload, send); + sent?; + assert_eq!(written.unwrap_err().code, Code::InvalidArgument); + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[2].method(), "DELETE"); + Ok(()) +} + +#[nativelink_test] +async fn transient_head_retries_through_shared_store() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ + event( + StatusCode::SERVICE_UNAVAILABLE, + "ServiceUnavailable", + ), + ReplayEvent::new( + http::Request::builder().body(SdkBody::empty()).unwrap(), + http::Response::builder() + .status(StatusCode::OK) + .header("content-length", "5") + .body(SdkBody::empty()) + .unwrap(), + ), + ]); + assert_eq!( + store(mock.clone())?.has(StoreKey::new_str("key")).await?, + Some(5) + ); + assert_eq!(mock.actual_requests().count(), 2); + Ok(()) +} + +#[nativelink_test] +async fn streaming_read_requests_exact_inclusive_range() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event(StatusCode::PARTIAL_CONTENT, "bcd")]); + let store = store(mock.clone())?; + let (mut tx, mut rx) = make_buf_channel_pair(); + let read = store.get_part(StoreKey::new_str("key"), &mut tx, 1, Some(3)); + let ((), bytes) = futures::try_join!(read, rx.consume(None))?; + assert_eq!(bytes, "bcd"); + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!(requests[0].headers().get("range"), Some("bytes=1-3")); + Ok(()) +} + +#[nativelink_test] +async fn zero_length_read_makes_no_request() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![]); + let bytes = store(mock.clone())? + .get_part_unchunked(StoreKey::new_str("key"), 1, Some(0)) + .await?; + assert!(bytes.is_empty()); + assert_eq!(mock.actual_requests().count(), 0); + Ok(()) +} + +#[nativelink_test] +async fn endpoint_override_and_temporary_credentials_reach_the_wire() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![event(StatusCode::NOT_FOUND, "")]); + let spec = ExperimentalOssSpec { + endpoint: Some("https://s3.oss-cn-hangzhou-internal.aliyuncs.com".into()), + session_token: Some("TESTTOKEN".into()), + ..spec() + }; + let store = OssStore::new_with_http_client(&spec, mock.clone(), NOW)?; + assert_eq!(store.has(StoreKey::new_str("key")).await?, None); + let requests: Vec<_> = mock.actual_requests().collect(); + assert_eq!( + requests[0].uri(), + "https://test-bucket.s3.oss-cn-hangzhou-internal.aliyuncs.com/cas/key" + ); + assert_eq!( + requests[0].headers().get("x-amz-security-token"), + Some("TESTTOKEN") + ); + Ok(()) +} + +#[nativelink_test] +async fn exact_size_error_does_not_publish_an_object() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![]); + let store = store(mock.clone())?; + let (mut tx, rx) = make_buf_channel_pair(); + let upload = store.update(StoreKey::new_str("key"), rx, UploadSizeInfo::ExactSize(4)); + let send = async { + tx.send("abc".into()).await?; + tx.send_eof() + }; + let (written, sent) = tokio::join!(upload, send); + sent?; + assert_eq!(written.unwrap_err().code, Code::InvalidArgument); + assert_eq!(mock.actual_requests().count(), 0); + Ok(()) +} + +#[derive(Clone, Debug)] +struct GatedPartClient { + gate: Arc, + started: Arc, + active: Arc, + peak: Arc, + parts: Arc, +} + +impl aws_smithy_runtime_api::client::http::HttpConnector for GatedPartClient { + fn call( + &self, + request: aws_smithy_runtime_api::client::orchestrator::HttpRequest, + ) -> aws_smithy_runtime_api::client::http::HttpConnectorFuture { + use std::sync::atomic::Ordering; + let client = self.clone(); + aws_smithy_runtime_api::client::http::HttpConnectorFuture::new(async move { + let response = if request.uri().contains("partNumber=") { + client.parts.fetch_add(1, Ordering::SeqCst); + let active = client.active.fetch_add(1, Ordering::SeqCst) + 1; + client.peak.fetch_max(active, Ordering::SeqCst); + client.started.notify_one(); + client.gate.acquire().await.unwrap().forget(); + client.active.fetch_sub(1, Ordering::SeqCst); + http::Response::builder() + .header("etag", "part-etag") + .body(SdkBody::empty()) + .unwrap() + } else if request.uri().contains("uploads") { + http::Response::builder().body(SdkBody::from("upload-1")).unwrap() + } else { + http::Response::builder().body(SdkBody::from("object-etag")).unwrap() + }; + Ok(response.try_into().unwrap()) + }) + } +} + +impl aws_smithy_runtime_api::client::http::HttpClient for GatedPartClient { + fn http_connector( + &self, + _: &aws_smithy_runtime_api::client::http::HttpConnectorSettings, + _: &aws_smithy_runtime_api::client::runtime_components::RuntimeComponents, + ) -> aws_smithy_runtime_api::client::http::SharedHttpConnector { + aws_smithy_runtime_api::client::http::SharedHttpConnector::new(self.clone()) + } +} + +#[nativelink_test] +async fn slow_backend_bounds_in_flight_multipart_requests() -> Result<(), Error> { + use std::sync::atomic::{AtomicUsize, Ordering}; + const PARTS: usize = 12; + const PART_BYTES: usize = 5 * 1024 * 1024; + let client = GatedPartClient { + gate: Arc::new(tokio::sync::Semaphore::new(0)), + started: Arc::new(tokio::sync::Notify::new()), + active: Arc::new(AtomicUsize::new(0)), + peak: Arc::new(AtomicUsize::new(0)), + parts: Arc::new(AtomicUsize::new(0)), + }; + let store = OssStore::new_with_http_client(&spec(), client.clone(), NOW)?; + let (mut tx, rx) = make_buf_channel_pair(); + let upload = spawn!("oss_multipart_upload", async move { + store + .update( + StoreKey::new_str("key"), + rx, + UploadSizeInfo::ExactSize((PARTS * PART_BYTES) as u64), + ) + .await + }); + let send = spawn!("oss_multipart_producer", async move { + let chunk = bytes::Bytes::from(vec![1; PART_BYTES]); + for _ in 0..PARTS { + tx.send(chunk.clone()).await?; + } + tx.send_eof() + }); + client.started.notified().await; + // Give the producer and upload loop opportunities to fill the limit while + // the connector deliberately keeps all part requests pending. + for _ in 0..1024 { + tokio::task::yield_now().await; + } + let peak_while_blocked = client.peak.load(Ordering::SeqCst); + client.gate.add_permits(PARTS); + let written = upload.await.unwrap()?; + send.await.unwrap()?; + assert_eq!(written, (PARTS * PART_BYTES) as u64); + assert_eq!(client.parts.load(Ordering::SeqCst), PARTS); + assert!(peak_while_blocked > 0 && peak_while_blocked <= 2); + assert!(client.peak.load(Ordering::SeqCst) <= 2); + Ok(()) +} + +#[test] +fn missing_store_credentials_are_rejected() { + for (access_key_id, secret_access_key) in [("", "SECRET"), ("KEY", ""), (" ", "SECRET")] { + let error = OssStore::build_aws_spec(&ExperimentalOssSpec { + access_key_id: access_key_id.into(), + secret_access_key: secret_access_key.into(), + ..spec() + }) + .unwrap_err(); + assert_eq!(error.code, Code::InvalidArgument); + assert!(!error.to_string().contains("SECRET")); + } +} + +#[nativelink_test] +async fn configured_credentials_are_isolated_between_stores() -> Result<(), Error> { + let mock = StaticReplayClient::new(vec![ + event(StatusCode::NOT_FOUND, ""), + event(StatusCode::NOT_FOUND, ""), + ]); + let first = OssStore::new_with_http_client( + &ExperimentalOssSpec { + access_key_id: "OSS_FIRST_KEY".into(), + secret_access_key: "OSS_FIRST_SECRET".into(), + ..spec() + }, + mock.clone(), + NOW, + )?; + let second = OssStore::new_with_http_client( + &ExperimentalOssSpec { + access_key_id: "OSS_SECOND_KEY".into(), + secret_access_key: "OSS_SECOND_SECRET".into(), + ..spec() + }, + mock.clone(), + NOW, + )?; + assert_eq!(first.has(StoreKey::new_str("key")).await?, None); + assert_eq!(second.has(StoreKey::new_str("key")).await?, None); + let requests: Vec<_> = mock.actual_requests().collect(); + assert!( + requests[0] + .headers() + .get("authorization") + .unwrap() + .contains("Credential=OSS_FIRST_KEY/") + ); + assert!( + requests[1] + .headers() + .get("authorization") + .unwrap() + .contains("Credential=OSS_SECOND_KEY/") + ); + Ok(()) +} diff --git a/nativelink-store/tests/s3_store_test.rs b/nativelink-store/tests/s3_store_test.rs index c2a25b35d4e..b91aa18ad8f 100644 --- a/nativelink-store/tests/s3_store_test.rs +++ b/nativelink-store/tests/s3_store_test.rs @@ -326,7 +326,7 @@ async fn smoke_test_get_part() -> Result<(), Error> { .uri(format!( "https://{BUCKET_NAME}.s3.{REGION}.amazonaws.com/{VALID_HASH1}-{AC_ENTRY_SIZE}?x-id=GetObject", )) - .header("range", format!("bytes={}-{}", OFFSET, OFFSET + LENGTH)) + .header("range", format!("bytes={}-{}", OFFSET, OFFSET + LENGTH - 1)) .body(SdkBody::empty()) .unwrap(), http::Response::builder() @@ -564,7 +564,7 @@ async fn ensure_empty_string_in_stream_works_test() -> Result<(), Error> { .uri(format!( "https://{BUCKET_NAME}.s3.{REGION}.amazonaws.com/{VALID_HASH1}-{CAS_ENTRY_SIZE}?x-id=GetObject", )) - .header("range", format!("bytes={}-{}", 0, CAS_ENTRY_SIZE)) + .header("range", format!("bytes={}-{}", 0, CAS_ENTRY_SIZE - 1)) .body(SdkBody::empty()) .unwrap(), http::Response::builder() @@ -882,7 +882,7 @@ async fn multipart_chunk_size_clamp_min() -> Result<(), Error> { ), ]); let test_config = Builder::new() - .behavior_version(BehaviorVersion::v2026_01_12()) + .behavior_version(BehaviorVersion::latest()) .region(Region::from_static(REGION)) .http_client(mock_client.clone()) .build(); diff --git a/web/apps/docs/content/docs/configuration/stores.mdx b/web/apps/docs/content/docs/configuration/stores.mdx index df4fb340b33..1ffca2c4805 100644 --- a/web/apps/docs/content/docs/configuration/stores.mdx +++ b/web/apps/docs/content/docs/configuration/stores.mdx @@ -218,7 +218,7 @@ specific backends (object storage, Redis, OCI registries) live in Cloud object storage is one store type, `experimental_cloud_object_store`, with a `provider` field selecting between `aws`, `gcs`, `azure`, `ontap`, - `r2`, and `oci`. Configs and older docs showing a top-level `s3` key predate + `r2`, `oci`, and (in development builds) `oss`. Configs and older docs showing a top-level `s3` key predate that consolidation and will not parse. diff --git a/web/apps/docs/content/docs/how-to/stores/meta.json b/web/apps/docs/content/docs/how-to/stores/meta.json index d6dcf8cb97f..e8cdd4034b9 100644 --- a/web/apps/docs/content/docs/how-to/stores/meta.json +++ b/web/apps/docs/content/docs/how-to/stores/meta.json @@ -7,6 +7,7 @@ "redis", "mongo", "oci-object-storage", + "oss", "compose-stores", "compression", "chunking-and-dedup" diff --git a/web/apps/docs/content/docs/how-to/stores/oss.mdx b/web/apps/docs/content/docs/how-to/stores/oss.mdx new file mode 100644 index 00000000000..67e589c6555 --- /dev/null +++ b/web/apps/docs/content/docs/how-to/stores/oss.mdx @@ -0,0 +1,163 @@ +--- +title: Alibaba Cloud OSS +description: Use an OSS bucket as the cache through the S3-compatible API. +--- + + + A NativeLink development build with the `oss` provider, an OSS bucket, and an + Alibaba Cloud AccessKey with object read, write, and multipart abort permissions. + + +The `oss` provider uses the [OSS S3-compatible API](https://www.alibabacloud.com/help/en/oss/developer-reference/use-aws-sdks-to-access-oss) +and the shared NativeLink S3 store. It derives the service endpoint from the +bucket region, uses virtual-hosted addressing, and signs requests with AWS SigV4. +Optional trailing checksums are disabled to avoid `aws-chunked` uploads. +OSS lists the required object and multipart operations in its +[S3 compatibility reference](https://www.alibabacloud.com/help/en/oss/developer-reference/compatibility-with-amazon-s3). +This is an API compatibility basis, not evidence of a successful live deployment. + +This provider is available in development builds. Object names are `key_prefix` +followed by the standard NativeLink store key, as in the +[S3 store](/how-to/stores/s3-and-compatible). + +## Configure the store [#configure-the-store] + +Configure credentials explicitly for each OSS store. Reference independent +variables such as `OSS_ACCESS_KEY_ID` and `OSS_ACCESS_KEY_SECRET` through shell +expansion. For temporary credentials, also configure `session_token`, for example +`${OSS_SESSION_TOKEN}`. Keep the values out of source control. + +The provider requires explicit credentials and ignores the AWS credential chain +and `AWS_*` variables, so an AWS S3 store in the same process can use its own +credentials. Different OSS stores can reference different variable names. +Automatic Alibaba Cloud RAM role discovery is not supported. + +```json5 +stores: [ + { + name: "CAS_MAIN_STORE", + experimental_cloud_object_store: { + provider: "oss", + region: "cn-hangzhou", + bucket: "example-build-cache", + access_key_id: "${OSS_ACCESS_KEY_ID}", + secret_access_key: "${OSS_ACCESS_KEY_SECRET}", + key_prefix: "cas/", + }, + }, +] +``` + +Use a separate `key_prefix`, such as `ac/`, for the Action Cache. The optional +`endpoint` overrides the service URL, for example +`https://s3.oss-cn-hangzhou-internal.aliyuncs.com`. It must be HTTPS and must not +contain a bucket, object path, user information, query, or fragment. A native OSS +REST endpoint (`oss-...`) is not interchangeable with an S3-compatible endpoint +(`s3.oss-...`). + +## Use a custom domain [#use-a-custom-domain] + +Set `cname: true` to use an HTTPS custom domain bound to the bucket. This option +requires an explicit `endpoint`. The provider uses that host directly, without +adding the bucket name to the host or path. The default is `cname: false`, which +keeps virtual-hosted service-endpoint addressing. + +```json5 +stores: [ + { + name: "CAS_MAIN_STORE", + experimental_cloud_object_store: { + provider: "oss", + region: "cn-hangzhou", + bucket: "example-build-cache", + access_key_id: "${OSS_ACCESS_KEY_ID}", + secret_access_key: "${OSS_ACCESS_KEY_SECRET}", + endpoint: "https://cache.example.com", + cname: true, + key_prefix: "cas/", + }, + }, +] +``` + +An object named `cas/key` uses `https://cache.example.com/cas/key`. Keep `bucket` +set to the actual bucket and `region` set to its region. Requests still use AWS +SigV4, with the custom host included in the signature. This mode does not switch +to the native OSS REST protocol. + +Before starting NativeLink, [bind the custom domain to the bucket and configure +DNS](https://www.alibabacloud.com/help/en/oss/user-guide/access-buckets-via-custom-domain-names), +then configure a valid TLS certificate for the domain. Setting `cname` does not +create the binding, DNS record, or certificate. Mainland buckets require a domain +with ICP registration. The endpoint must not contain an object path, credentials, +query, or fragment. Confirm S3 API compatibility against the real domain before +deployment; wire replay tests do not verify the OSS domain binding or policy. + +## Endpoint access restrictions [#endpoint-access-restrictions] + +Alibaba Cloud restricts default public endpoints for accounts that activated OSS +on or after March 20, 2025, at 00:00 UTC+8, when the bucket is in a Chinese mainland +region. Data operations can fail with `PublicEndpointForbidden` (HTTP 400). +The [OSS policy notice](https://www.alibabacloud.com/en/notice/oss_update_notice_policy_change_in_calling_data_api_operations_via_the_default_public_domain_name_45a) +describes the affected operations. The +[AWS SDK access guide](https://www.alibabacloud.com/help/en/oss/developer-reference/use-aws-sdks-to-access-oss) +also lists this restriction; the S3-compatible endpoint is not a workaround. + +For affected accounts accessing a bucket over the public internet, Alibaba Cloud +requires a custom domain name (CNAME). Use the [custom-domain mode](#use-a-custom-domain) +with an endpoint bound to the bucket. Setting `endpoint` alone keeps the default +service-endpoint addressing and adds the bucket name to the host. + +For a workload in an Alibaba Cloud VPC with access to the bucket's regional +internal endpoint, use the +[documented internal-endpoint option](https://www.alibabacloud.com/help/en/oss/user-guide/0048-00000401). +For a bucket in `cn-hangzhou`, set this field in the OSS store configuration: + +```json5 +endpoint: "https://s3.oss-cn-hangzhou-internal.aliyuncs.com", +``` + +The internal endpoint is not reachable from the public internet. Accounts that +activated OSS before the policy cutoff and buckets outside Chinese mainland +regions are outside the scope of this notice. Confirm the account, bucket region, +and network access before choosing an endpoint. + +## Retry and resource behavior [#retry-and-resource-behavior] + +The optional `retry` object uses the standard store retry policy. SDK retries +are disabled, but the shared HTTP transport also retries: its attempts can +multiply the store retry loop. A total retry budget is not yet implemented. + +Each SDK operation attempt has a 120-second timeout. The custom HTTP connector +does not currently apply the SDK connection-timeout setting to TCP connections. +Receiving an upload part before an SDK call, reading a download body after the +response headers, and waiting for a consumer are not covered by a total transfer +deadline. + +Uploads and downloads use the existing S3 transfer implementation, including +multipart uploads and transfer size validation. Its buffering and concurrency +behavior is inherited. `multipart_max_concurrent_uploads` bounds both queued +parts and concurrent part requests per upload (default: 2). Part sizes depend +on the declared upload size. This adapter does not provide a global byte budget +or upload admission queue. OSS-specific spool, TTL, and read-proxy settings are +not supported. To expire cached objects, configure bucket lifecycle rules +separately. + +## Verify [#verify] + +Point the CAS service at this store and run the same build twice with remote +caching enabled. The first build should upload its outputs and the second should +reuse them. Check both a small Action Cache entry and an object large enough to +use multipart upload. A successful replay test does not establish compatibility +with a live bucket, its permissions, or its endpoint restrictions. + + + The second build reports remote cache hits, and downloaded outputs match the + first build. OSS wire replay tests cover endpoint selection, signing scope, + direct reads, small uploads, multipart completion, and failure cleanup; + verification against a real OSS bucket is still required before production use. + + + + Add verification or a local fast tier around the OSS backend. + diff --git a/web/apps/docs/content/docs/how-to/stores/s3-and-compatible.mdx b/web/apps/docs/content/docs/how-to/stores/s3-and-compatible.mdx index 7017639fd30..e2639b1a199 100644 --- a/web/apps/docs/content/docs/how-to/stores/s3-and-compatible.mdx +++ b/web/apps/docs/content/docs/how-to/stores/s3-and-compatible.mdx @@ -1,6 +1,6 @@ --- title: S3 and compatible -description: "Back the cache with Amazon S3, Cloudflare R2 or NetApp ONTAP S3: one store type, three providers, and the shared options that decide cost and failure behaviour." +description: "Back the cache with Amazon S3, Cloudflare R2, Alibaba Cloud OSS, or NetApp ONTAP S3, and choose the options that decide cost and failure behaviour." --- {/* archetype: how-to, see /web/apps/docs/templates/how-to.mdx */} @@ -14,22 +14,25 @@ behind a local fast tier, with credentials that aren't in the config file. A config file you can edit (see [Configuration](/configuration)), and a bucket you can write to. -Three providers share one store type and one block of common options. `aws` -and `r2` share one implementation (R2 is the S3 store with a derived endpoint -and region `auto`); `ontap` has its own, with a few differences noted below. -They differ mainly in how the endpoint and credentials are derived: +Four S3-compatible providers use the `experimental_cloud_object_store` type. +`aws`, `r2`, and `oss` share the S3 transfer implementation; `ontap` has its own. +R2 derives its endpoint from an account ID and uses region `auto`. OSS is available +in development builds and has its own configuration fields rather than the common +options used by the other three providers. The providers differ in how they +derive endpoints and credentials: | Provider | `provider` value | Identifies the endpoint by | | --- | --- | --- | | Amazon S3 | `aws` | `region` + `bucket` | | Cloudflare R2 | `r2` | `account_id` (endpoint is `https://{account_id}.r2.cloudflarestorage.com`) | +| Alibaba Cloud OSS | `oss` | `region` (endpoint is `https://s3.oss-{region}.aliyuncs.com`), an explicit service `endpoint`, or a bucket-bound custom domain with `cname: true` | | NetApp ONTAP S3 | `ontap` | an explicit `endpoint` plus `vserver_name` | [GCS](/how-to/stores/gcs), [Azure Blob](/how-to/stores/azure) and [OCI Object Storage](/how-to/stores/oci-object-storage) use the same store type with their own `provider` values and have their own pages. -In development builds, S3, R2, and ONTAP validate small uploads before issuing a PUT and multipart +In development builds, S3, R2, OSS, and ONTAP validate small uploads before issuing a PUT and multipart uploads before completion. A small upload is buffered through EOF before its request starts; multipart uploads continue to stream parts. See [upload integrity](/how-to/stores/compose-stores#upload-integrity) for the shared @@ -74,8 +77,10 @@ rules can then treat them differently. ### Other services that speak S3 [#other-services-that-speak-s3] -The `aws` provider reaches any S3-compatible service, not only AWS. Four -optional fields cover the difference: +The `aws` provider can reach other S3-compatible services. For Alibaba Cloud OSS, +use the dedicated [`oss` provider](#alibaba-cloud-oss), which configures its +addressing and checksum requirements. Four optional fields configure other +services: | Field | Set it when | | --- | --- | @@ -124,6 +129,23 @@ you can get away with. +## Alibaba Cloud OSS [#alibaba-cloud-oss] + +The `oss` provider uses Alibaba Cloud's S3-compatible API with virtual-hosted +addressing and explicit credentials for each store. It requires `region`, +`bucket`, `access_key_id`, and `secret_access_key`; it does not fall back to the +AWS credential chain. Optional fields include `session_token`, `endpoint`, `cname`, +`key_prefix`, `retry`, and `multipart_max_concurrent_uploads` (default: 2). + +Follow the [OSS guide](/how-to/stores/oss) for configuration and verification. +Check its [endpoint access restrictions](/how-to/stores/oss#endpoint-access-restrictions) +before choosing a public endpoint. Affected accounts with Chinese mainland +buckets need an internal endpoint reachable from their Alibaba Cloud VPC or a +bucket-bound custom domain with `cname: true` and an explicit HTTPS `endpoint`. +Verify S3 API compatibility against the real custom domain before deployment. +The common-options block below applies to `aws`, `r2`, and `ontap`, not +`oss`. + ## NetApp ONTAP S3 [#netapp-ontap-s3] On-premises S3 needs an explicit endpoint, the storage VM serving it, and @@ -183,10 +205,10 @@ often the slowest thing in the build, and the answer is usually stale-tolerant. -## The options all three share [#the-options-all-three-share] +## Common options for AWS, R2, and ONTAP [#the-options-all-three-share] -Every provider flattens the same block of common options. These are the ones -worth setting deliberately: +The `aws`, `r2`, and `ontap` providers flatten the same block of common options. +These are the ones worth setting deliberately: | Field | Default | What it decides | | --- | --- | --- | @@ -285,6 +307,8 @@ For `aws`, the credential chain found nothing. Confirm the process can see `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, or that the instance/task role is attached and reachable. For `r2`, set `access_key_id` and `secret_access_key` explicitly, because R2 tokens are not in the AWS chain. +The `oss` provider requires explicit OSS credentials for each store; see the +[OSS credential configuration](/how-to/stores/oss#configure-the-store). diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index cf692ce1a34..7810dbe3747 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (3472f873) + Source: nativelink-config @ main (8ec59189) Regenerate from web/: pnpm run --filter @nativelink/docs gen:config-reference */} @@ -229,6 +229,28 @@ It supports the following backends: } ``` +7. **Alibaba Cloud OSS:** + Uses the OSS S3-compatible endpoint and AWS `SigV4`. Supply the OSS + access key explicitly per store, for example by shell-expanding + `OSS_ACCESS_KEY_ID` and `OSS_ACCESS_KEY_SECRET` in the config. + Set `cname` to true with an explicit HTTPS `endpoint` to use a custom + domain bound to the bucket, without adding the bucket to the host or + path. The default is false. Configure the domain binding, DNS, and + TLS certificate before starting the store. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oss", + "region": "cn-hangzhou", + "bucket": "example-build-cache", + "access_key_id": "oss_access_key_id", + "secret_access_key": "oss_secret_access_key", + "cname": false, + "key_prefix": "cas/" + } + ``` + **Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) ### `ontap_s3_existence_cache` diff --git a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx index 77809fd9c00..e242aa25e6c 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx @@ -60,7 +60,7 @@ store name the server exposes. | --- | --- | --- | | `memory` | An in-process hash map | Fastest, gone on restart. Good for AC tiers and dev. | | `filesystem` | Local disk | Survives restarts; scans and rebuilds its index on startup. | -| `experimental_cloud_object_store` | A cloud object bucket | One store type, six `provider`s: `aws` (S3), `gcs`, `azure`, `ontap` (NetApp ONTAP S3), `r2` (Cloudflare), `oci` (Oracle Cloud Infrastructure). All speak an S3-compatible API except `gcs` and `azure`. | +| `experimental_cloud_object_store` | A cloud object bucket | One store type with `provider`s: `aws` (S3), `gcs`, `azure`, `ontap` (NetApp ONTAP S3), `r2` (Cloudflare), `oci` (Oracle Cloud Infrastructure), and `oss` (Alibaba Cloud OSS, development builds). All speak an S3-compatible API except `gcs` and `azure`. | | `redis_store` | Any Redis-API-compatible service | Pairs well with `size_partitioning`; most Redis services cap uploads around 256 to 512 MB. | | `experimental_mongo` | MongoDB | Supports CAS and scheduler data, with optional change streams for scheduler subscriptions. | | `grpc` | Another NativeLink-compatible gRPC endpoint | Proxies calls upstream. Useful for a regional cache in front of a central cluster. |