diff --git a/src/common/storage/src/config.rs b/src/common/storage/src/config.rs index 7afe9bb284406..b85b5abb73f48 100644 --- a/src/common/storage/src/config.rs +++ b/src/common/storage/src/config.rs @@ -68,7 +68,7 @@ impl Default for CacheConfig { fn default() -> Self { Self { num_cpus: 0, - params: StorageParams::Moka(StorageMokaConfig::default()), + params: StorageParams::None, } } } @@ -91,6 +91,11 @@ pub enum StorageParams { Oss(StorageOssConfig), S3(StorageS3Config), Redis(StorageRedisConfig), + + /// None means this storage type is none. + /// + /// This type is mostly for cache which mean bypass the cache logic. + None, } impl Default for StorageParams { @@ -153,6 +158,9 @@ impl Display for StorageParams { v.db, v.root, v.endpoint_url ) } + StorageParams::None => { + write!(f, "none",) + } } } } @@ -177,6 +185,7 @@ impl StorageParams { StorageParams::S3(v) => v.endpoint_url.starts_with("https://"), StorageParams::Gcs(v) => v.endpoint_url.starts_with("https://"), StorageParams::Redis(_) => false, + StorageParams::None => false, } } } diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index 4d2e75a1400db..434b75ab27b75 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -13,10 +13,13 @@ // limitations under the License. use std::env; +use std::io::Error; +use std::io::ErrorKind; use std::io::Result; use std::ops::Deref; use std::time::Duration; +use anyhow::anyhow; use backon::ExponentialBackoff; use common_base::base::GlobalIORuntime; use common_base::base::Singleton; @@ -72,6 +75,12 @@ pub fn init_operator(cfg: &StorageParams) -> Result { StorageParams::S3(cfg) => init_s3_operator(cfg)?, StorageParams::Oss(cfg) => init_oss_operator(cfg)?, StorageParams::Redis(cfg) => init_redis_operator(cfg)?, + v => { + return Err(Error::new( + ErrorKind::InvalidInput, + anyhow!("Unsupported storage type: {:?}", v), + )); + } }; let op = op @@ -412,15 +421,7 @@ impl DataOperator { /// background auto evict at any time. #[derive(Clone, Debug)] pub struct CacheOperator { - op: Operator, -} - -impl Deref for CacheOperator { - type Target = Operator; - - fn deref(&self) -> &Self::Target { - &self.op - } + op: Option, } static CACHE_OPERATOR: OnceCell> = OnceCell::new(); @@ -437,6 +438,10 @@ impl CacheOperator { } pub async fn try_create(conf: &CacheConfig) -> common_exception::Result { + if conf.params == StorageParams::None { + return Ok(CacheOperator { op: None }); + } + let operator = init_operator(&conf.params)?; // OpenDAL will send a real request to underlying storage to check whether it works or not. @@ -456,13 +461,17 @@ impl CacheOperator { ))); } - Ok(CacheOperator { op: operator }) + Ok(CacheOperator { op: Some(operator) }) } - pub fn instance() -> CacheOperator { + pub fn instance() -> Option { match CACHE_OPERATOR.get() { None => panic!("CacheOperator is not init"), - Some(op) => op.get(), + Some(op) => op.get().inner(), } } + + fn inner(&self) -> Option { + self.op.clone() + } } diff --git a/src/query/config/src/outer_v0.rs b/src/query/config/src/outer_v0.rs index 86e74be45099d..5859d3488b161 100644 --- a/src/query/config/src/outer_v0.rs +++ b/src/query/config/src/outer_v0.rs @@ -518,6 +518,9 @@ impl From for CacheConfig { }; match inner.params { + StorageParams::None => { + cfg.cache_type = "none".to_string(); + } StorageParams::Fs(v) => { cfg.cache_type = "fs".to_string(); cfg.fs = v.into(); @@ -544,6 +547,7 @@ impl TryInto for CacheConfig { num_cpus: self.cache_num_cpus, params: { match self.cache_type.as_str() { + "none" => StorageParams::None, "fs" => StorageParams::Fs(self.fs.try_into()?), "moka" => StorageParams::Moka(self.moka.try_into()?), "redis" => StorageParams::Redis(self.redis.try_into()?), diff --git a/src/query/service/tests/it/configs.rs b/src/query/service/tests/it/configs.rs index 1c3b0cda7328a..a555f52178adb 100644 --- a/src/query/service/tests/it/configs.rs +++ b/src/query/service/tests/it/configs.rs @@ -159,7 +159,7 @@ endpoint_url = "" root = "" [storage.cache] -type = "moka" +type = "none" num_cpus = 0 [storage.cache.fs] diff --git a/src/query/service/tests/it/storages/testdata/system-tables.txt b/src/query/service/tests/it/storages/testdata/system-tables.txt index 70e08e4f92464..e20f3eae3a7c3 100644 --- a/src/query/service/tests/it/storages/testdata/system-tables.txt +++ b/src/query/service/tests/it/storages/testdata/system-tables.txt @@ -240,7 +240,7 @@ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemCo | storage | cache.redis.password | | | | storage | cache.redis.root | | | | storage | cache.redis.username | | | -| storage | cache.type | moka | | +| storage | cache.type | none | | | storage | fs.data_path | _data | | | storage | gcs.bucket | | | | storage | gcs.credential | | | diff --git a/src/query/storages/fuse/fuse/src/fuse_table.rs b/src/query/storages/fuse/fuse/src/fuse_table.rs index 72fd476897153..d62a8201831e8 100644 --- a/src/query/storages/fuse/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/fuse/src/fuse_table.rs @@ -15,7 +15,6 @@ use std::any::Any; use std::collections::HashMap; use std::convert::TryFrom; -use std::ops::Deref; use std::str; use std::sync::Arc; @@ -115,12 +114,14 @@ impl FuseTable { } }; let data_metrics = Arc::new(StorageMetrics::default()); - operator = operator - .layer(StorageMetricsLayer::new(data_metrics.clone())) - .layer(ContentCacheLayer::new( - CacheOperator::instance().deref().clone(), + operator = operator.layer(StorageMetricsLayer::new(data_metrics.clone())); + // If cache op is valid, layered with ContentCacheLayer. + if let Some(cache_op) = CacheOperator::instance() { + operator = operator.layer(ContentCacheLayer::new( + cache_op, ContentCacheStrategy::Fixed(1024 * 1024), )); + } Ok(Box::new(FuseTable { table_info,