Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/styles/config/vocabularies/TraceMachina/accept.txt
Original file line number Diff line number Diff line change
Expand Up @@ -499,3 +499,7 @@ simulatable
URIs
[Aa]bliterated
[Uu]npatched
Alibaba
OSS
AccessKey
SigV4
40 changes: 26 additions & 14 deletions nativelink-config/examples/stores-config.json5
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,18 @@
},
{
name: "8",
"experimental_cloud_object_store": {
"provider": "oss",
"region": "cn-hangzhou",
"bucket": "example-build-cache",
"access_key_id": "oss_access_key_id",
"secret_access_key": "oss_secret_access_key",
"cname": false,
"key_prefix": "cas/"
}
},
{
name: "9",
"ontap_s3_existence_cache": {
"index_path": "/path/to/cache/index.json",
"sync_interval_seconds": 300,
Expand All @@ -129,7 +141,7 @@
}
},
{
name: "9",
name: "10",
"verify": {
"backend": {
"memory": {
Expand All @@ -143,7 +155,7 @@
}
},
{
name: "10",
name: "11",
"completeness_checking": {
"backend": {
"filesystem": {
Expand All @@ -162,7 +174,7 @@
}
},
{
name: "11",
name: "12",
"compression": {
"compression_algorithm": {
"lz4": {}
Expand All @@ -179,7 +191,7 @@
}
},
{
name: "12",
name: "13",
"dedup": {
"index_store": {
"memory": {
Expand Down Expand Up @@ -218,7 +230,7 @@
}
},
{
name: "13",
name: "14",
"existence_cache": {
"backend": {
"memory": {
Expand All @@ -234,7 +246,7 @@
}
},
{
name: "14",
name: "15",
"fast_slow": {
"fast": {
"filesystem": {
Expand All @@ -257,7 +269,7 @@
}
},
{
name: "15",
name: "16",
"shard": {
"stores": [
{
Expand All @@ -273,7 +285,7 @@
}
},
{
name: "16",
name: "17",
"filesystem": {
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
Expand All @@ -283,13 +295,13 @@
}
},
{
name: "17",
name: "18",
"ref_store": {
"name": "FS_CONTENT_STORE"
}
},
{
name: "18",
name: "19",
"size_partitioning": {
"size": "128mib",
"lower_store": {
Expand All @@ -306,7 +318,7 @@
}
},
{
name: "19",
name: "20",
"grpc": {
"instance_name": "main",
"endpoints": [
Expand All @@ -327,7 +339,7 @@
}
},
{
name: "20",
name: "21",
"redis_store": {
"addresses": [
"redis://127.0.0.1:6379/",
Expand All @@ -336,11 +348,11 @@
}
},
{
name: "21",
name: "22",
"noop": {}
},
{
name: "22",
name: "23",
"experimental_mongo": {
"connection_string": "mongodb://localhost:27017",
"database": "nativelink",
Expand Down
101 changes: 101 additions & 0 deletions nativelink-config/src/stores.rs
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,28 @@ pub enum StoreSpec {
/// }
/// }
/// ```
///
/// 7. **Alibaba Cloud OSS:**
/// Uses the OSS S3-compatible endpoint and AWS `SigV4`. Supply the OSS
/// access key explicitly per store, for example by shell-expanding
/// `OSS_ACCESS_KEY_ID` and `OSS_ACCESS_KEY_SECRET` in the config.
/// Set `cname` to true with an explicit HTTPS `endpoint` to use a custom
/// domain bound to the bucket, without adding the bucket to the host or
/// path. The default is false. Configure the domain binding, DNS, and
/// TLS certificate before starting the store.
///
/// **Example JSON Config:**
/// ```json
/// "experimental_cloud_object_store": {
/// "provider": "oss",
/// "region": "cn-hangzhou",
/// "bucket": "example-build-cache",
/// "access_key_id": "oss_access_key_id",
/// "secret_access_key": "oss_secret_access_key",
/// "cname": false,
/// "key_prefix": "cas/"
/// }
/// ```
ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec),

/// ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store
Expand Down Expand Up @@ -948,6 +970,84 @@ pub struct ExperimentalOciSpec {
pub common: CommonObjectSpec,
}

/// Alibaba Cloud OSS through its S3-compatible API.
#[derive(Serialize, Deserialize, Default, Clone)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
pub struct ExperimentalOssSpec {
/// OSS region, for example `cn-hangzhou`. Used for AWS `SigV4` signing
/// and to derive the default S3-compatible endpoint.
#[serde(deserialize_with = "convert_string_with_shellexpand")]
pub region: String,

/// Bucket name to use as the backend.
#[serde(deserialize_with = "convert_string_with_shellexpand")]
pub bucket: String,

/// Access key ID for this store. Required; no AWS credential-chain fallback.
/// Use shell expansion, for example `${OSS_ACCESS_KEY_ID}`.
#[serde(deserialize_with = "convert_string_with_shellexpand")]
pub access_key_id: String,

/// Access key secret for this store. Use `${OSS_ACCESS_KEY_SECRET}` to
/// keep it out of the config file. Independent of AWS S3 store credentials.
#[serde(deserialize_with = "convert_string_with_shellexpand")]
pub secret_access_key: String,

/// Optional STS token for this store, for example `${OSS_SESSION_TOKEN}`.
#[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")]
pub session_token: Option<String>,

/// Optional HTTPS S3-compatible service endpoint, without a bucket or path.
/// Default: `https://s3.oss-{region}.aliyuncs.com`.
/// With `cname: true`, this must be the custom domain bound to the bucket.
/// Native OSS REST endpoints are not interchangeable with this endpoint.
#[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")]
pub endpoint: Option<String>,

/// Use `endpoint` as the bucket's custom domain without adding the bucket
/// to the host or path. Requires an explicit HTTPS endpoint, an OSS domain
/// binding, and a valid certificate for that domain. Default: false.
#[serde(default)]
pub cname: bool,

/// Prefix prepended verbatim to each object key, for example `cas/`.
#[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")]
pub key_prefix: Option<String>,

/// Retry policy for the shared S3 store. SDK retries are disabled to avoid
/// multiplying attempts. Default: the standard store retry policy.
#[serde(default)]
pub retry: Retry,

/// Maximum concurrent part requests per upload. Must be positive.
/// Default: 2. Queued parts are bounded by the same count; this is not
/// a global byte budget. Part size is chosen by the shared S3 store.
#[serde(
default,
deserialize_with = "convert_optional_numeric_with_shellexpand"
)]
pub multipart_max_concurrent_uploads: Option<usize>,
}

impl core::fmt::Debug for ExperimentalOssSpec {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("ExperimentalOssSpec")
.field("region", &self.region)
.field("bucket", &self.bucket)
.field("endpoint", &self.endpoint)
.field("cname", &self.cname)
.field("key_prefix", &self.key_prefix)
.field("retry", &self.retry)
.field(
"multipart_max_concurrent_uploads",
&self.multipart_max_concurrent_uploads,
)
.field("credentials", &"<redacted>")
.finish()
}
}

#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(deny_unknown_fields)]
#[cfg_attr(feature = "dev-schema", derive(JsonSchema))]
Expand Down Expand Up @@ -1266,6 +1366,7 @@ pub enum ExperimentalCloudObjectSpec {
Ontap(ExperimentalOntapS3Spec),
R2(ExperimentalR2Spec),
Oci(ExperimentalOciSpec),
Oss(ExperimentalOssSpec),
}

impl Default for ExperimentalCloudObjectSpec {
Expand Down
65 changes: 65 additions & 0 deletions nativelink-config/tests/deserialization_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -796,3 +796,68 @@ fn http_listener_tcp_nodelay_defaults_and_overrides() {
assert!(serde_json5::from_str::<HttpListener>(&input).is_err());
}
}

#[test]
fn oss_provider_accepts_minimal_config_and_rejects_unrelated_options() {
use nativelink_config::stores::{ExperimentalCloudObjectSpec, StoreSpec};
let config = r#"{"experimental_cloud_object_store":{"provider":"oss","region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"OSS_TEST_KEY","secret_access_key":"OSS_TEST_SECRET"}}"#;
let parsed: StoreSpec = serde_json5::from_str(config).unwrap();
assert!(matches!(
parsed,
StoreSpec::ExperimentalCloudObjectStore(ExperimentalCloudObjectSpec::Oss(_))
));
for field in [
"consider_expired_after_s",
"max_retry_buffer_per_request",
"max_concurrent_uploads",
"upload_queue_timeout_s",
] {
let config = format!(
r#"{{"provider":"oss","region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"OSS_TEST_KEY","secret_access_key":"OSS_TEST_SECRET","{field}":1}}"#
);
assert!(
serde_json5::from_str::<ExperimentalCloudObjectSpec>(&config).is_err(),
"unexpected OSS field accepted: {field}"
);
}
}

#[test]
fn oss_credentials_are_required_and_redacted_from_debug() {
use nativelink_config::stores::ExperimentalOssSpec;
let missing = r#"{"region":"cn-hangzhou","bucket":"test-bucket"}"#;
assert!(serde_json5::from_str::<ExperimentalOssSpec>(missing).is_err());
let spec = ExperimentalOssSpec {
access_key_id: "PRIVATE_OSS_KEY_ID".into(),
secret_access_key: "PRIVATE_OSS_SECRET".into(),
session_token: Some("PRIVATE_OSS_TOKEN".into()),
..Default::default()
};
let debug = format!("{spec:?}");
for credential in [
"PRIVATE_OSS_KEY_ID",
"PRIVATE_OSS_SECRET",
"PRIVATE_OSS_TOKEN",
] {
assert!(!debug.contains(credential));
}
}

#[test]
fn oss_cname_defaults_to_false_and_round_trips() {
use nativelink_config::stores::ExperimentalOssSpec;

let config = r#"{"region":"cn-hangzhou","bucket":"test-bucket","access_key_id":"TESTKEY","secret_access_key":"TESTSECRET"}"#;
let mut spec: ExperimentalOssSpec = serde_json5::from_str(config).unwrap();
assert!(!spec.cname);
spec.cname = true;
spec.endpoint = Some("https://cache.example.com".into());
let round_trip: ExperimentalOssSpec =
serde_json::from_value(serde_json::to_value(&spec).unwrap()).unwrap();
assert!(round_trip.cname);
assert_eq!(round_trip.endpoint, spec.endpoint);
for invalid in ["null", "1", "\"true\""] {
let input = format!("{},\"cname\":{invalid}}}", &config[..config.len() - 1]);
assert!(serde_json5::from_str::<ExperimentalOssSpec>(&input).is_err());
}
}
2 changes: 2 additions & 0 deletions nativelink-store/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ rust_library(
"src/oci_store.rs",
"src/ontap_s3_existence_cache_store.rs",
"src/ontap_s3_store.rs",
"src/oss_store.rs",
"src/r2_store.rs",
"src/redis_store.rs",
"src/redis_utils/aggregate_types.rs",
Expand Down Expand Up @@ -163,6 +164,7 @@ rust_test_suite(
"tests/oci_store_test.rs",
"tests/ontap_s3_existence_cache_store_test.rs",
"tests/ontap_s3_store_test.rs",
"tests/oss_store_test.rs",
"tests/r2_store_test.rs",
"tests/redis_store_test.rs",
"tests/ref_store_test.rs",
Expand Down
4 changes: 4 additions & 0 deletions nativelink-store/src/default_store_factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ use crate::noop_store::NoopStore;
use crate::oci_store::OciStore;
use crate::ontap_s3_existence_cache_store::OntapS3ExistenceCache;
use crate::ontap_s3_store::OntapS3Store;
use crate::oss_store::OssStore;
use crate::r2_store::R2Store;
use crate::redis_store::RedisStore;
use crate::ref_store::RefStore;
Expand Down Expand Up @@ -78,6 +79,9 @@ pub fn store_factory<'a>(
ExperimentalCloudObjectSpec::R2(r2_config) => {
R2Store::new(r2_config, SystemTime::now).await?
}
ExperimentalCloudObjectSpec::Oss(oss_config) => {
OssStore::new(oss_config, SystemTime::now)?
}
ExperimentalCloudObjectSpec::Oci(oci_config) => {
OciStore::new(oci_config, SystemTime::now).await?
}
Expand Down
1 change: 1 addition & 0 deletions nativelink-store/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ pub mod noop_store;
pub mod oci_store;
pub mod ontap_s3_existence_cache_store;
pub mod ontap_s3_store;
pub mod oss_store;
pub mod r2_store;
pub mod redis_store;
mod redis_utils;
Expand Down
Loading
Loading