diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c79c5498aa..e6398735a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ * [ENHANCEMENT] Compactor: Optimize cleaner run time. #6815 * [ENHANCEMENT] Parquet Storage: Allow percentage based dynamic shard size for Parquet Converter. #6817 * [ENHANCEMENT] Query Frontend: Enhance the performance of the JSON codec. #6816 +* [ENHANCEMENT] Metadata Cache: Support inmemory and multi level cache backend. #6829 * [ENHANCEMENT] Store Gateway: Allow to ignore syncing blocks older than certain time using `ignore_blocks_before`. #6830 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517 * [BUGFIX] Ingester: Fix labelset data race condition. #6573 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 60e2ff65208..38387a6406e 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -854,8 +854,8 @@ blocks_storage: [backend: | default = ""] inmemory: - # Maximum size in bytes of in-memory chunk cache used to speed up chunk - # lookups (shared between all tenants). + # Maximum size in bytes of in-memory chunks cache used (shared between + # all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: | default = 1073741824] @@ -1095,11 +1095,19 @@ blocks_storage: [subrange_ttl: | default = 24h] metadata_cache: - # Backend for metadata cache, if not empty. Supported values: memcached, - # redis, and '' (disable). + # The metadata cache backend type. Single or Multiple cache backend can be + # provided. Supported values in single cache: memcached, redis, inmemory, + # and '' (disable). Supported values in multi level cache: a + # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: | default = ""] + inmemory: + # Maximum size in bytes of in-memory metadata cache used (shared between + # all tenants). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes + [max_size_bytes: | default = 1073741824] + memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV @@ -1301,6 +1309,21 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: | default = 0.05] + multilevel: + # The maximum number of concurrent asynchronous operations can occur + # when backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency + [max_async_concurrency: | default = 3] + + # The maximum number of enqueued asynchronous operations allowed when + # backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size + [max_async_buffer_size: | default = 10000] + + # The maximum number of items to backfill per asynchronous operation. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items + [max_backfill_items: | default = 10000] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 730f17f1e59..5752584e6b3 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -975,8 +975,8 @@ blocks_storage: [backend: | default = ""] inmemory: - # Maximum size in bytes of in-memory chunk cache used to speed up chunk - # lookups (shared between all tenants). + # Maximum size in bytes of in-memory chunks cache used (shared between + # all tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: | default = 1073741824] @@ -1216,11 +1216,19 @@ blocks_storage: [subrange_ttl: | default = 24h] metadata_cache: - # Backend for metadata cache, if not empty. Supported values: memcached, - # redis, and '' (disable). + # The metadata cache backend type. Single or Multiple cache backend can be + # provided. Supported values in single cache: memcached, redis, inmemory, + # and '' (disable). Supported values in multi level cache: a + # comma-separated list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: | default = ""] + inmemory: + # Maximum size in bytes of in-memory metadata cache used (shared between + # all tenants). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes + [max_size_bytes: | default = 1073741824] + memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV @@ -1422,6 +1430,21 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: | default = 0.05] + multilevel: + # The maximum number of concurrent asynchronous operations can occur + # when backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency + [max_async_concurrency: | default = 3] + + # The maximum number of enqueued asynchronous operations allowed when + # backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size + [max_async_buffer_size: | default = 10000] + + # The maximum number of items to backfill per asynchronous operation. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items + [max_backfill_items: | default = 10000] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index a2ff797a7d7..53719366d97 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -1450,8 +1450,8 @@ bucket_store: [backend: | default = ""] inmemory: - # Maximum size in bytes of in-memory chunk cache used to speed up chunk - # lookups (shared between all tenants). + # Maximum size in bytes of in-memory chunks cache used (shared between all + # tenants). # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes [max_size_bytes: | default = 1073741824] @@ -1688,11 +1688,19 @@ bucket_store: [subrange_ttl: | default = 24h] metadata_cache: - # Backend for metadata cache, if not empty. Supported values: memcached, - # redis, and '' (disable). + # The metadata cache backend type. Single or Multiple cache backend can be + # provided. Supported values in single cache: memcached, redis, inmemory, + # and '' (disable). Supported values in multi level cache: a comma-separated + # list of (inmemory, memcached, redis) # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend [backend: | default = ""] + inmemory: + # Maximum size in bytes of in-memory metadata cache used (shared between + # all tenants). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes + [max_size_bytes: | default = 1073741824] + memcached: # Comma separated list of memcached addresses. Supported prefixes are: # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, @@ -1892,6 +1900,21 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent [failure_percent: | default = 0.05] + multilevel: + # The maximum number of concurrent asynchronous operations can occur when + # backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency + [max_async_concurrency: | default = 3] + + # The maximum number of enqueued asynchronous operations allowed when + # backfilling cache items. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size + [max_async_buffer_size: | default = 10000] + + # The maximum number of items to backfill per asynchronous operation. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items + [max_backfill_items: | default = 10000] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/pkg/storage/tsdb/caching_bucket.go b/pkg/storage/tsdb/caching_bucket.go index eebce973c80..8be0411f717 100644 --- a/pkg/storage/tsdb/caching_bucket.go +++ b/pkg/storage/tsdb/caching_bucket.go @@ -26,11 +26,10 @@ import ( ) var ( - supportedChunkCacheBackends = []string{CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis} - supportedMetadataCacheBackends = []string{CacheBackendMemcached, CacheBackendRedis} + supportedBucketCacheBackends = []string{CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis} - errUnsupportedChunkCacheBackend = errors.New("unsupported chunk cache backend") - errDuplicatedChunkCacheBackend = errors.New("duplicated chunk cache backend") + errUnsupportedBucketCacheBackend = errors.New("unsupported cache backend") + errDuplicatedBucketCacheBackend = errors.New("duplicated cache backend") ) const ( @@ -39,36 +38,16 @@ const ( CacheBackendInMemory = "inmemory" ) -type MetadataCacheBackend struct { - Backend string `yaml:"backend"` - Memcached MemcachedClientConfig `yaml:"memcached"` - Redis RedisClientConfig `yaml:"redis"` +type BucketCacheBackend struct { + Backend string `yaml:"backend"` + InMemory InMemoryBucketCacheConfig `yaml:"inmemory"` + Memcached MemcachedClientConfig `yaml:"memcached"` + Redis RedisClientConfig `yaml:"redis"` + MultiLevel MultiLevelBucketCacheConfig `yaml:"multilevel"` } // Validate the config. -func (cfg *MetadataCacheBackend) Validate() error { - switch cfg.Backend { - case CacheBackendMemcached: - return cfg.Memcached.Validate() - case CacheBackendRedis: - return cfg.Redis.Validate() - case "": - default: - return fmt.Errorf("unsupported cache backend: %s", cfg.Backend) - } - return nil -} - -type ChunkCacheBackend struct { - Backend string `yaml:"backend"` - InMemory InMemoryChunkCacheConfig `yaml:"inmemory"` - Memcached MemcachedClientConfig `yaml:"memcached"` - Redis RedisClientConfig `yaml:"redis"` - MultiLevel MultiLevelChunkCacheConfig `yaml:"multilevel"` -} - -// Validate the config. -func (cfg *ChunkCacheBackend) Validate() error { +func (cfg *BucketCacheBackend) Validate() error { if cfg.Backend == "" { return nil } @@ -83,12 +62,12 @@ func (cfg *ChunkCacheBackend) Validate() error { } for _, backend := range splitBackends { - if !util.StringsContain(supportedChunkCacheBackends, backend) { - return errUnsupportedChunkCacheBackend + if !util.StringsContain(supportedBucketCacheBackends, backend) { + return errUnsupportedBucketCacheBackend } if _, ok := configuredBackends[backend]; ok { - return errDuplicatedChunkCacheBackend + return errDuplicatedBucketCacheBackend } switch backend { @@ -110,7 +89,7 @@ func (cfg *ChunkCacheBackend) Validate() error { } type ChunksCacheConfig struct { - ChunkCacheBackend `yaml:",inline"` + BucketCacheBackend `yaml:",inline"` SubrangeSize int64 `yaml:"subrange_size"` MaxGetRangeRequests int `yaml:"max_get_range_requests"` @@ -121,11 +100,11 @@ type ChunksCacheConfig struct { func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("The chunks cache backend type. Single or Multiple cache backend can be provided. "+ "Supported values in single cache: %s, %s, %s, and '' (disable). "+ - "Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedChunkCacheBackends, ", "))) + "Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedBucketCacheBackends, ", "))) cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.") cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.") - cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.") + cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.", "chunks") cfg.MultiLevel.RegisterFlagsWithPrefix(f, prefix+"multilevel.") f.Int64Var(&cfg.SubrangeSize, prefix+"subrange-size", 16000, "Size of each subrange that bucket object is split into for better caching.") @@ -138,18 +117,18 @@ func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix st } func (cfg *ChunksCacheConfig) Validate() error { - return cfg.ChunkCacheBackend.Validate() + return cfg.BucketCacheBackend.Validate() } -type InMemoryChunkCacheConfig struct { +type InMemoryBucketCacheConfig struct { MaxSizeBytes uint64 `yaml:"max_size_bytes"` } -func (cfg *InMemoryChunkCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { - f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), "Maximum size in bytes of in-memory chunk cache used to speed up chunk lookups (shared between all tenants).") +func (cfg *InMemoryBucketCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string, item string) { + f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), fmt.Sprintf("Maximum size in bytes of in-memory %s cache used (shared between all tenants).", item)) } -func (cfg *InMemoryChunkCacheConfig) toInMemoryChunkCacheConfig() cache.InMemoryCacheConfig { +func (cfg *InMemoryBucketCacheConfig) toInMemoryCacheConfig() cache.InMemoryCacheConfig { maxCacheSize := model.Bytes(cfg.MaxSizeBytes) // Calculate the max item size. @@ -165,7 +144,7 @@ func (cfg *InMemoryChunkCacheConfig) toInMemoryChunkCacheConfig() cache.InMemory } type MetadataCacheConfig struct { - MetadataCacheBackend `yaml:",inline"` + BucketCacheBackend `yaml:",inline"` TenantsListTTL time.Duration `yaml:"tenants_list_ttl"` TenantBlocksListTTL time.Duration `yaml:"tenant_blocks_list_ttl"` @@ -181,10 +160,14 @@ type MetadataCacheConfig struct { } func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { - f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("Backend for metadata cache, if not empty. Supported values: %s, and '' (disable).", strings.Join(supportedMetadataCacheBackends, ", "))) + f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("The metadata cache backend type. Single or Multiple cache backend can be provided. "+ + "Supported values in single cache: %s, %s, %s, and '' (disable). "+ + "Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedBucketCacheBackends, ", "))) cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.") cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.") + cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.", "metadata") + cfg.MultiLevel.RegisterFlagsWithPrefix(f, prefix+"multilevel.") f.DurationVar(&cfg.TenantsListTTL, prefix+"tenants-list-ttl", 15*time.Minute, "How long to cache list of tenants in the bucket.") f.DurationVar(&cfg.TenantBlocksListTTL, prefix+"tenant-blocks-list-ttl", 5*time.Minute, "How long to cache list of blocks for each tenant.") @@ -200,14 +183,14 @@ func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix } func (cfg *MetadataCacheConfig) Validate() error { - return cfg.MetadataCacheBackend.Validate() + return cfg.BucketCacheBackend.Validate() } func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, matchers Matchers, bkt objstore.InstrumentedBucket, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error) { cfg := cache.NewCachingBucketConfig() cachingConfigured := false - chunksCache, err := createChunkCache("chunks-cache", &chunksConfig.ChunkCacheBackend, logger, reg) + chunksCache, err := createBucketCache("chunks-cache", &chunksConfig.BucketCacheBackend, logger, reg) if err != nil { return nil, errors.Wrapf(err, "chunks-cache") } @@ -218,7 +201,7 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata cfg.CacheGetRange("parquet-chunks", chunksCache, matchers.GetParquetChunksMatcher(), chunksConfig.SubrangeSize, chunksConfig.AttributesTTL, chunksConfig.SubrangeTTL, chunksConfig.MaxGetRangeRequests) } - metadataCache, err := createMetadataCache("metadata-cache", &metadataConfig.MetadataCacheBackend, logger, reg) + metadataCache, err := createBucketCache("metadata-cache", &metadataConfig.BucketCacheBackend, logger, reg) if err != nil { return nil, errors.Wrapf(err, "metadata-cache") } @@ -255,7 +238,7 @@ func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner cfg := cache.NewCachingBucketConfig() cachingConfigured := false - metadataCache, err := createMetadataCache("metadata-cache", &metadataConfig.MetadataCacheBackend, logger, reg) + metadataCache, err := createBucketCache("metadata-cache", &metadataConfig.BucketCacheBackend, logger, reg) if err != nil { return nil, errors.Wrapf(err, "metadata-cache") } @@ -287,32 +270,7 @@ func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner return storecache.NewCachingBucket(bkt, cfg, logger, reg) } -func createMetadataCache(cacheName string, cacheBackend *MetadataCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) { - switch cacheBackend.Backend { - case "": - // No caching. - return nil, nil - case CacheBackendMemcached: - var client cacheutil.MemcachedClient - client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, cacheBackend.Memcached.ToMemcachedClientConfig(), reg) - if err != nil { - return nil, errors.Wrapf(err, "failed to create memcached client") - } - return cache.NewMemcachedCache(cacheName, logger, client, reg), nil - - case CacheBackendRedis: - redisCache, err := cacheutil.NewRedisClientWithConfig(logger, cacheName, cacheBackend.Redis.ToRedisClientConfig(), reg) - if err != nil { - return nil, errors.Wrapf(err, "failed to create redis client") - } - return cache.NewRedisCache(cacheName, logger, redisCache, reg), nil - - default: - return nil, errors.Errorf("unsupported cache type for cache %s: %s", cacheName, cacheBackend.Backend) - } -} - -func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) { +func createBucketCache(cacheName string, cacheBackend *BucketCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) { if cacheBackend.Backend == "" { // No caching. return nil, nil @@ -326,7 +284,7 @@ func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger for _, backend := range splitBackends { switch backend { case CacheBackendInMemory: - inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, reg, cacheBackend.InMemory.toInMemoryChunkCacheConfig()) + inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, reg, cacheBackend.InMemory.toInMemoryCacheConfig()) if err != nil { return nil, errors.Wrapf(err, "failed to create in-memory chunk cache") } @@ -347,7 +305,7 @@ func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger } } - return newMultiLevelChunkCache(cacheName, cacheBackend.MultiLevel, reg, caches...), nil + return newMultiLevelBucketCache(cacheName, cacheBackend.MultiLevel, reg, caches...), nil } type Matchers struct { diff --git a/pkg/storage/tsdb/caching_bucket_test.go b/pkg/storage/tsdb/caching_bucket_test.go index 875134452e9..92f86a70ec9 100644 --- a/pkg/storage/tsdb/caching_bucket_test.go +++ b/pkg/storage/tsdb/caching_bucket_test.go @@ -8,25 +8,25 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_ChunkCacheBackendValidation(t *testing.T) { +func Test_BucketCacheBackendValidation(t *testing.T) { tests := map[string]struct { - cfg ChunkCacheBackend + cfg BucketCacheBackend expectedErr error }{ - "valid chunk cache type ('')": { - cfg: ChunkCacheBackend{ + "valid bucket cache type ('')": { + cfg: BucketCacheBackend{ Backend: "", }, expectedErr: nil, }, - "valid chunk cache type (in-memory)": { - cfg: ChunkCacheBackend{ + "valid bucket cache type (in-memory)": { + cfg: BucketCacheBackend{ Backend: CacheBackendInMemory, }, expectedErr: nil, }, - "valid chunk cache type (memcached)": { - cfg: ChunkCacheBackend{ + "valid bucket cache type (memcached)": { + cfg: BucketCacheBackend{ Backend: CacheBackendMemcached, Memcached: MemcachedClientConfig{ Addresses: "dns+localhost:11211", @@ -34,8 +34,8 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { }, expectedErr: nil, }, - "valid chunk cache type (redis)": { - cfg: ChunkCacheBackend{ + "valid bucket cache type (redis)": { + cfg: BucketCacheBackend{ Backend: CacheBackendRedis, Redis: RedisClientConfig{ Addresses: "localhost:6379", @@ -43,14 +43,14 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { }, expectedErr: nil, }, - "invalid chunk cache type": { - cfg: ChunkCacheBackend{ + "invalid bucket cache type": { + cfg: BucketCacheBackend{ Backend: "dummy", }, - expectedErr: errUnsupportedChunkCacheBackend, + expectedErr: errUnsupportedBucketCacheBackend, }, - "valid multi chunk cache type": { - cfg: ChunkCacheBackend{ + "valid multi bucket cache type": { + cfg: BucketCacheBackend{ Backend: fmt.Sprintf("%s,%s,%s", CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis), Memcached: MemcachedClientConfig{ Addresses: "dns+localhost:11211", @@ -58,7 +58,7 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { Redis: RedisClientConfig{ Addresses: "localhost:6379", }, - MultiLevel: MultiLevelChunkCacheConfig{ + MultiLevel: MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 1, MaxAsyncBufferSize: 1, MaxBackfillItems: 1, @@ -66,24 +66,24 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { }, expectedErr: nil, }, - "duplicate multi chunk cache type": { - cfg: ChunkCacheBackend{ + "duplicate multi bucket cache type": { + cfg: BucketCacheBackend{ Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendInMemory), - MultiLevel: MultiLevelChunkCacheConfig{ + MultiLevel: MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 1, MaxAsyncBufferSize: 1, MaxBackfillItems: 1, }, }, - expectedErr: errDuplicatedChunkCacheBackend, + expectedErr: errDuplicatedBucketCacheBackend, }, "invalid max async concurrency": { - cfg: ChunkCacheBackend{ + cfg: BucketCacheBackend{ Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached), Memcached: MemcachedClientConfig{ Addresses: "dns+localhost:11211", }, - MultiLevel: MultiLevelChunkCacheConfig{ + MultiLevel: MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 0, MaxAsyncBufferSize: 1, MaxBackfillItems: 1, @@ -92,12 +92,12 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { expectedErr: errInvalidMaxAsyncConcurrency, }, "invalid max async buffer size": { - cfg: ChunkCacheBackend{ + cfg: BucketCacheBackend{ Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached), Memcached: MemcachedClientConfig{ Addresses: "dns+localhost:11211", }, - MultiLevel: MultiLevelChunkCacheConfig{ + MultiLevel: MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 1, MaxAsyncBufferSize: 0, MaxBackfillItems: 1, @@ -106,12 +106,12 @@ func Test_ChunkCacheBackendValidation(t *testing.T) { expectedErr: errInvalidMaxAsyncBufferSize, }, "invalid max back fill items": { - cfg: ChunkCacheBackend{ + cfg: BucketCacheBackend{ Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached), Memcached: MemcachedClientConfig{ Addresses: "dns+localhost:11211", }, - MultiLevel: MultiLevelChunkCacheConfig{ + MultiLevel: MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 1, MaxAsyncBufferSize: 1, MaxBackfillItems: 0, diff --git a/pkg/storage/tsdb/multilevel_chunk_cache.go b/pkg/storage/tsdb/multilevel_bucket_cache.go similarity index 68% rename from pkg/storage/tsdb/multilevel_chunk_cache.go rename to pkg/storage/tsdb/multilevel_bucket_cache.go index 8daa2f56ce9..8358c22ced0 100644 --- a/pkg/storage/tsdb/multilevel_chunk_cache.go +++ b/pkg/storage/tsdb/multilevel_bucket_cache.go @@ -4,6 +4,7 @@ import ( "context" "errors" "flag" + "fmt" "time" "github.com/prometheus/client_golang/prometheus" @@ -12,7 +13,7 @@ import ( "github.com/thanos-io/thanos/pkg/cacheutil" ) -type multiLevelChunkCache struct { +type multiLevelBucketCache struct { name string caches []cache.Cache @@ -25,7 +26,7 @@ type multiLevelChunkCache struct { backfillTTL time.Duration } -type MultiLevelChunkCacheConfig struct { +type MultiLevelBucketCacheConfig struct { MaxAsyncConcurrency int `yaml:"max_async_concurrency"` MaxAsyncBufferSize int `yaml:"max_async_buffer_size"` MaxBackfillItems int `yaml:"max_backfill_items"` @@ -33,7 +34,7 @@ type MultiLevelChunkCacheConfig struct { BackFillTTL time.Duration `yaml:"-"` } -func (cfg *MultiLevelChunkCacheConfig) Validate() error { +func (cfg *MultiLevelBucketCacheConfig) Validate() error { if cfg.MaxAsyncBufferSize <= 0 { return errInvalidMaxAsyncBufferSize } @@ -46,45 +47,58 @@ func (cfg *MultiLevelChunkCacheConfig) Validate() error { return nil } -func (cfg *MultiLevelChunkCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { +func (cfg *MultiLevelBucketCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.IntVar(&cfg.MaxAsyncConcurrency, prefix+"max-async-concurrency", 3, "The maximum number of concurrent asynchronous operations can occur when backfilling cache items.") f.IntVar(&cfg.MaxAsyncBufferSize, prefix+"max-async-buffer-size", 10000, "The maximum number of enqueued asynchronous operations allowed when backfilling cache items.") f.IntVar(&cfg.MaxBackfillItems, prefix+"max-backfill-items", 10000, "The maximum number of items to backfill per asynchronous operation.") } -func newMultiLevelChunkCache(name string, cfg MultiLevelChunkCacheConfig, reg prometheus.Registerer, c ...cache.Cache) cache.Cache { +func newMultiLevelBucketCache(name string, cfg MultiLevelBucketCacheConfig, reg prometheus.Registerer, c ...cache.Cache) cache.Cache { if len(c) == 1 { return c[0] } - return &multiLevelChunkCache{ + itemName := "" + metricHelpText := "" + switch name { + case "chunks-cache": + itemName = "chunks_cache" + metricHelpText = "chunks cache" + case "metadata-cache": + itemName = "metadata_cache" + metricHelpText = "metadata cache" + default: + itemName = name + } + + return &multiLevelBucketCache{ name: name, caches: c, backfillProcessor: cacheutil.NewAsyncOperationProcessor(cfg.MaxAsyncBufferSize, cfg.MaxAsyncConcurrency), fetchLatency: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "cortex_store_multilevel_chunks_cache_fetch_duration_seconds", - Help: "Histogram to track latency to fetch items from multi level chunk cache", + Name: fmt.Sprintf("cortex_store_multilevel_%s_fetch_duration_seconds", itemName), + Help: fmt.Sprintf("Histogram to track latency to fetch items from multi level %s", metricHelpText), Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 10, 15, 20, 25, 30, 40, 50, 60, 90}, }, nil), backFillLatency: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "cortex_store_multilevel_chunks_cache_backfill_duration_seconds", - Help: "Histogram to track latency to backfill items from multi level chunk cache", + Name: fmt.Sprintf("cortex_store_multilevel_%s_backfill_duration_seconds", itemName), + Help: fmt.Sprintf("Histogram to track latency to backfill items from multi level %s", metricHelpText), Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 10, 15, 20, 25, 30, 40, 50, 60, 90}, }, nil), storeDroppedItems: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_store_multilevel_chunks_cache_backfill_dropped_items_total", - Help: "Total number of items dropped due to async buffer full when backfilling multilevel cache ", + Name: fmt.Sprintf("cortex_store_multilevel_%s_backfill_dropped_items_total", itemName), + Help: fmt.Sprintf("Total number of items dropped due to async buffer full when backfilling multilevel %s", metricHelpText), }), backfillDroppedItems: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_store_multilevel_chunks_cache_store_dropped_items_total", - Help: "Total number of items dropped due to async buffer full when storing multilevel cache ", + Name: fmt.Sprintf("cortex_store_multilevel_%s_store_dropped_items_total", itemName), + Help: fmt.Sprintf("Total number of items dropped due to async buffer full when storing multilevel %s", metricHelpText), }), maxBackfillItems: cfg.MaxBackfillItems, backfillTTL: cfg.BackFillTTL, } } -func (m *multiLevelChunkCache) Store(data map[string][]byte, ttl time.Duration) { +func (m *multiLevelBucketCache) Store(data map[string][]byte, ttl time.Duration) { for _, c := range m.caches { if err := m.backfillProcessor.EnqueueAsync(func() { c.Store(data, ttl) @@ -94,7 +108,7 @@ func (m *multiLevelChunkCache) Store(data map[string][]byte, ttl time.Duration) } } -func (m *multiLevelChunkCache) Fetch(ctx context.Context, keys []string) map[string][]byte { +func (m *multiLevelBucketCache) Fetch(ctx context.Context, keys []string) map[string][]byte { timer := prometheus.NewTimer(m.fetchLatency.WithLabelValues()) defer timer.ObserveDuration() @@ -157,6 +171,6 @@ func (m *multiLevelChunkCache) Fetch(ctx context.Context, keys []string) map[str return hits } -func (m *multiLevelChunkCache) Name() string { +func (m *multiLevelBucketCache) Name() string { return m.name } diff --git a/pkg/storage/tsdb/multilevel_chunk_cache_test.go b/pkg/storage/tsdb/multilevel_bucket_cache_test.go similarity index 81% rename from pkg/storage/tsdb/multilevel_chunk_cache_test.go rename to pkg/storage/tsdb/multilevel_bucket_cache_test.go index 4b50b8fd028..f63ad95460c 100644 --- a/pkg/storage/tsdb/multilevel_chunk_cache_test.go +++ b/pkg/storage/tsdb/multilevel_bucket_cache_test.go @@ -12,9 +12,9 @@ import ( "github.com/thanos-io/thanos/pkg/cache" ) -func Test_MultiLevelChunkCacheStore(t *testing.T) { +func Test_MultiLevelBucketCacheStore(t *testing.T) { ttl := time.Hour * 24 - cfg := MultiLevelChunkCacheConfig{ + cfg := MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 10, MaxAsyncBufferSize: 100000, MaxBackfillItems: 10000, @@ -58,13 +58,13 @@ func Test_MultiLevelChunkCacheStore(t *testing.T) { } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - m1 := newMockChunkCache("m1", tc.m1InitData) - m2 := newMockChunkCache("m2", tc.m2InitData) + m1 := newMockBucketCache("m1", tc.m1InitData) + m2 := newMockBucketCache("m2", tc.m2InitData) reg := prometheus.NewRegistry() - c := newMultiLevelChunkCache("chunk-cache", cfg, reg, m1, m2) + c := newMultiLevelBucketCache("chunks-cache", cfg, reg, m1, m2) c.Store(tc.storeData, ttl) - mlc := c.(*multiLevelChunkCache) + mlc := c.(*multiLevelBucketCache) // Wait until async operation finishes. mlc.backfillProcessor.Stop() @@ -74,8 +74,8 @@ func Test_MultiLevelChunkCacheStore(t *testing.T) { } } -func Test_MultiLevelChunkCacheFetchRace(t *testing.T) { - cfg := MultiLevelChunkCacheConfig{ +func Test_MultiLevelBucketCacheFetchRace(t *testing.T) { + cfg := MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 10, MaxAsyncBufferSize: 100000, MaxBackfillItems: 10000, @@ -83,7 +83,7 @@ func Test_MultiLevelChunkCacheFetchRace(t *testing.T) { } reg := prometheus.NewRegistry() - m1 := newMockChunkCache("m1", map[string][]byte{ + m1 := newMockBucketCache("m1", map[string][]byte{ "key1": []byte("value1"), "key2": []byte("value2"), }) @@ -96,7 +96,7 @@ func Test_MultiLevelChunkCacheFetchRace(t *testing.T) { "key3": []byte("value3"), }, time.Minute) - c := newMultiLevelChunkCache("chunk-cache", cfg, reg, inMemory, m1) + c := newMultiLevelBucketCache("chunks-cache", cfg, reg, inMemory, m1) hits := c.Fetch(context.Background(), []string{"key1", "key2", "key3", "key4"}) @@ -105,14 +105,14 @@ func Test_MultiLevelChunkCacheFetchRace(t *testing.T) { // We should be able to change the returned values without any race problem delete(hits, "key1") - mlc := c.(*multiLevelChunkCache) + mlc := c.(*multiLevelBucketCache) //Wait until async operation finishes. mlc.backfillProcessor.Stop() } -func Test_MultiLevelChunkCacheFetch(t *testing.T) { - cfg := MultiLevelChunkCacheConfig{ +func Test_MultiLevelBucketCacheFetch(t *testing.T) { + cfg := MultiLevelBucketCacheConfig{ MaxAsyncConcurrency: 10, MaxAsyncBufferSize: 100000, MaxBackfillItems: 10000, @@ -181,13 +181,13 @@ func Test_MultiLevelChunkCacheFetch(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - m1 := newMockChunkCache("m1", tc.m1ExistingData) - m2 := newMockChunkCache("m2", tc.m2ExistingData) + m1 := newMockBucketCache("m1", tc.m1ExistingData) + m2 := newMockBucketCache("m2", tc.m2ExistingData) reg := prometheus.NewRegistry() - c := newMultiLevelChunkCache("chunk-cache", cfg, reg, m1, m2) + c := newMultiLevelBucketCache("chunks-cache", cfg, reg, m1, m2) fetchData := c.Fetch(context.Background(), tc.fetchKeys) - mlc := c.(*multiLevelChunkCache) + mlc := c.(*multiLevelBucketCache) // Wait until async operation finishes. mlc.backfillProcessor.Stop() @@ -198,7 +198,7 @@ func Test_MultiLevelChunkCacheFetch(t *testing.T) { } } -type mockChunkCache struct { +type mockBucketCache struct { mu sync.Mutex name string data map[string][]byte @@ -206,22 +206,22 @@ type mockChunkCache struct { fetchedKeys []string } -func newMockChunkCache(name string, data map[string][]byte) *mockChunkCache { +func newMockBucketCache(name string, data map[string][]byte) *mockBucketCache { if data == nil { data = make(map[string][]byte) } - return &mockChunkCache{ + return &mockBucketCache{ name: name, data: data, } } -func (m *mockChunkCache) Store(data map[string][]byte, _ time.Duration) { +func (m *mockBucketCache) Store(data map[string][]byte, _ time.Duration) { m.data = data } -func (m *mockChunkCache) Fetch(_ context.Context, keys []string) map[string][]byte { +func (m *mockBucketCache) Fetch(_ context.Context, keys []string) map[string][]byte { m.mu.Lock() defer m.mu.Unlock() h := map[string][]byte{} @@ -236,6 +236,6 @@ func (m *mockChunkCache) Fetch(_ context.Context, keys []string) map[string][]by return h } -func (m *mockChunkCache) Name() string { +func (m *mockBucketCache) Name() string { return m.name } diff --git a/pkg/storage/tsdb/multilevel_cache.go b/pkg/storage/tsdb/multilevel_index_cache.go similarity index 100% rename from pkg/storage/tsdb/multilevel_cache.go rename to pkg/storage/tsdb/multilevel_index_cache.go diff --git a/pkg/storage/tsdb/multilevel_cache_test.go b/pkg/storage/tsdb/multilevel_index_cache_test.go similarity index 100% rename from pkg/storage/tsdb/multilevel_cache_test.go rename to pkg/storage/tsdb/multilevel_index_cache_test.go