Skip to content

Commit 8a7fccb

Browse files
yeya24yeya24
committed
unify chunks and metadata cache to bucket cache
Signed-off-by: yeya24 <[email protected]>
1 parent bf88f67 commit 8a7fccb

File tree

6 files changed

+114
-142
lines changed

6 files changed

+114
-142
lines changed

‎pkg/storage/tsdb/caching_bucket.go

Lines changed: 34 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,10 @@ import (
2626
)
2727

2828
var (
29-
supportedChunkCacheBackends = []string{CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis}
30-
supportedMetadataCacheBackends = []string{CacheBackendMemcached, CacheBackendRedis}
29+
supportedBucketCacheBackends = []string{CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis}
3130

32-
errUnsupportedChunkCacheBackend = errors.New("unsupported chunk cache backend")
33-
errDuplicatedChunkCacheBackend = errors.New("duplicated chunk cache backend")
31+
errUnsupportedBucketCacheBackend = errors.New("unsupported cache backend")
32+
errDuplicatedBucketCacheBackend = errors.New("duplicated cache backend")
3433
)
3534

3635
const (
@@ -39,36 +38,16 @@ const (
3938
CacheBackendInMemory = "inmemory"
4039
)
4140

42-
type MetadataCacheBackend struct {
43-
Backend string `yaml:"backend"`
44-
Memcached MemcachedClientConfig `yaml:"memcached"`
45-
Redis RedisClientConfig `yaml:"redis"`
41+
type BucketCacheBackend struct {
42+
Backend string `yaml:"backend"`
43+
InMemory InMemoryBucketCacheConfig `yaml:"inmemory"`
44+
Memcached MemcachedClientConfig `yaml:"memcached"`
45+
Redis RedisClientConfig `yaml:"redis"`
46+
MultiLevel MultiLevelBucketCacheConfig `yaml:"multilevel"`
4647
}
4748

4849
// Validate the config.
49-
func (cfg *MetadataCacheBackend) Validate() error {
50-
switch cfg.Backend {
51-
case CacheBackendMemcached:
52-
return cfg.Memcached.Validate()
53-
case CacheBackendRedis:
54-
return cfg.Redis.Validate()
55-
case "":
56-
default:
57-
return fmt.Errorf("unsupported cache backend: %s", cfg.Backend)
58-
}
59-
return nil
60-
}
61-
62-
type ChunkCacheBackend struct {
63-
Backend string `yaml:"backend"`
64-
InMemory InMemoryChunkCacheConfig `yaml:"inmemory"`
65-
Memcached MemcachedClientConfig `yaml:"memcached"`
66-
Redis RedisClientConfig `yaml:"redis"`
67-
MultiLevel MultiLevelChunkCacheConfig `yaml:"multilevel"`
68-
}
69-
70-
// Validate the config.
71-
func (cfg *ChunkCacheBackend) Validate() error {
50+
func (cfg *BucketCacheBackend) Validate() error {
7251
if cfg.Backend == "" {
7352
return nil
7453
}
@@ -83,12 +62,12 @@ func (cfg *ChunkCacheBackend) Validate() error {
8362
}
8463

8564
for _, backend := range splitBackends {
86-
if !util.StringsContain(supportedChunkCacheBackends, backend) {
87-
return errUnsupportedChunkCacheBackend
65+
if !util.StringsContain(supportedBucketCacheBackends, backend) {
66+
return errUnsupportedBucketCacheBackend
8867
}
8968

9069
if _, ok := configuredBackends[backend]; ok {
91-
return errDuplicatedChunkCacheBackend
70+
return errDuplicatedBucketCacheBackend
9271
}
9372

9473
switch backend {
@@ -110,7 +89,7 @@ func (cfg *ChunkCacheBackend) Validate() error {
11089
}
11190

11291
type ChunksCacheConfig struct {
113-
ChunkCacheBackend `yaml:",inline"`
92+
BucketCacheBackend `yaml:",inline"`
11493

11594
SubrangeSize int64 `yaml:"subrange_size"`
11695
MaxGetRangeRequests int `yaml:"max_get_range_requests"`
@@ -121,11 +100,11 @@ type ChunksCacheConfig struct {
121100
func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
122101
f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("The chunks cache backend type. Single or Multiple cache backend can be provided. "+
123102
"Supported values in single cache: %s, %s, %s, and '' (disable). "+
124-
"Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedChunkCacheBackends, ", ")))
103+
"Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedBucketCacheBackends, ", ")))
125104

126105
cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.")
127106
cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.")
128-
cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.")
107+
cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.", "chunks")
129108
cfg.MultiLevel.RegisterFlagsWithPrefix(f, prefix+"multilevel.")
130109

131110
f.Int64Var(&cfg.SubrangeSize, prefix+"subrange-size", 16000, "Size of each subrange that bucket object is split into for better caching.")
@@ -138,18 +117,18 @@ func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix st
138117
}
139118

140119
func (cfg *ChunksCacheConfig) Validate() error {
141-
return cfg.ChunkCacheBackend.Validate()
120+
return cfg.BucketCacheBackend.Validate()
142121
}
143122

144-
type InMemoryChunkCacheConfig struct {
123+
type InMemoryBucketCacheConfig struct {
145124
MaxSizeBytes uint64 `yaml:"max_size_bytes"`
146125
}
147126

148-
func (cfg *InMemoryChunkCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
149-
f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), "Maximum size in bytes of in-memory chunk cache used to speed up chunk lookups (shared between all tenants).")
127+
func (cfg *InMemoryBucketCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string, item string) {
128+
f.Uint64Var(&cfg.MaxSizeBytes, prefix+"max-size-bytes", uint64(1*units.Gibibyte), fmt.Sprintf("Maximum size in bytes of in-memory %s cache used (shared between all tenants).", item))
150129
}
151130

152-
func (cfg *InMemoryChunkCacheConfig) toInMemoryChunkCacheConfig() cache.InMemoryCacheConfig {
131+
func (cfg *InMemoryBucketCacheConfig) toInMemoryCacheConfig() cache.InMemoryCacheConfig {
153132
maxCacheSize := model.Bytes(cfg.MaxSizeBytes)
154133

155134
// Calculate the max item size.
@@ -165,7 +144,7 @@ func (cfg *InMemoryChunkCacheConfig) toInMemoryChunkCacheConfig() cache.InMemory
165144
}
166145

167146
type MetadataCacheConfig struct {
168-
MetadataCacheBackend `yaml:",inline"`
147+
BucketCacheBackend `yaml:",inline"`
169148

170149
TenantsListTTL time.Duration `yaml:"tenants_list_ttl"`
171150
TenantBlocksListTTL time.Duration `yaml:"tenant_blocks_list_ttl"`
@@ -181,10 +160,14 @@ type MetadataCacheConfig struct {
181160
}
182161

183162
func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
184-
f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("Backend for metadata cache, if not empty. Supported values: %s, and '' (disable).", strings.Join(supportedMetadataCacheBackends, ", ")))
163+
f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("The metadata cache backend type. Single or Multiple cache backend can be provided. "+
164+
"Supported values in single cache: %s, %s, %s, and '' (disable). "+
165+
"Supported values in multi level cache: a comma-separated list of (%s)", CacheBackendMemcached, CacheBackendRedis, CacheBackendInMemory, strings.Join(supportedBucketCacheBackends, ", ")))
185166

186167
cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.")
187168
cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.")
169+
cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.", "metadata")
170+
cfg.MultiLevel.RegisterFlagsWithPrefix(f, prefix+"multilevel.")
188171

189172
f.DurationVar(&cfg.TenantsListTTL, prefix+"tenants-list-ttl", 15*time.Minute, "How long to cache list of tenants in the bucket.")
190173
f.DurationVar(&cfg.TenantBlocksListTTL, prefix+"tenant-blocks-list-ttl", 5*time.Minute, "How long to cache list of blocks for each tenant.")
@@ -200,14 +183,14 @@ func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix
200183
}
201184

202185
func (cfg *MetadataCacheConfig) Validate() error {
203-
return cfg.MetadataCacheBackend.Validate()
186+
return cfg.BucketCacheBackend.Validate()
204187
}
205188

206189
func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig MetadataCacheConfig, matchers Matchers, bkt objstore.InstrumentedBucket, logger log.Logger, reg prometheus.Registerer) (objstore.InstrumentedBucket, error) {
207190
cfg := cache.NewCachingBucketConfig()
208191
cachingConfigured := false
209192

210-
chunksCache, err := createChunkCache("chunks-cache", &chunksConfig.ChunkCacheBackend, logger, reg)
193+
chunksCache, err := createBucketCache("chunks-cache", &chunksConfig.BucketCacheBackend, logger, reg)
211194
if err != nil {
212195
return nil, errors.Wrapf(err, "chunks-cache")
213196
}
@@ -218,7 +201,7 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata
218201
cfg.CacheGetRange("parquet-chunks", chunksCache, matchers.GetParquetChunksMatcher(), chunksConfig.SubrangeSize, chunksConfig.AttributesTTL, chunksConfig.SubrangeTTL, chunksConfig.MaxGetRangeRequests)
219202
}
220203

221-
metadataCache, err := createMetadataCache("metadata-cache", &metadataConfig.MetadataCacheBackend, logger, reg)
204+
metadataCache, err := createBucketCache("metadata-cache", &metadataConfig.BucketCacheBackend, logger, reg)
222205
if err != nil {
223206
return nil, errors.Wrapf(err, "metadata-cache")
224207
}
@@ -255,7 +238,7 @@ func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner
255238
cfg := cache.NewCachingBucketConfig()
256239
cachingConfigured := false
257240

258-
metadataCache, err := createMetadataCache("metadata-cache", &metadataConfig.MetadataCacheBackend, logger, reg)
241+
metadataCache, err := createBucketCache("metadata-cache", &metadataConfig.BucketCacheBackend, logger, reg)
259242
if err != nil {
260243
return nil, errors.Wrapf(err, "metadata-cache")
261244
}
@@ -287,32 +270,7 @@ func CreateCachingBucketForCompactor(metadataConfig MetadataCacheConfig, cleaner
287270
return storecache.NewCachingBucket(bkt, cfg, logger, reg)
288271
}
289272

290-
func createMetadataCache(cacheName string, cacheBackend *MetadataCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) {
291-
switch cacheBackend.Backend {
292-
case "":
293-
// No caching.
294-
return nil, nil
295-
case CacheBackendMemcached:
296-
var client cacheutil.MemcachedClient
297-
client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, cacheBackend.Memcached.ToMemcachedClientConfig(), reg)
298-
if err != nil {
299-
return nil, errors.Wrapf(err, "failed to create memcached client")
300-
}
301-
return cache.NewMemcachedCache(cacheName, logger, client, reg), nil
302-
303-
case CacheBackendRedis:
304-
redisCache, err := cacheutil.NewRedisClientWithConfig(logger, cacheName, cacheBackend.Redis.ToRedisClientConfig(), reg)
305-
if err != nil {
306-
return nil, errors.Wrapf(err, "failed to create redis client")
307-
}
308-
return cache.NewRedisCache(cacheName, logger, redisCache, reg), nil
309-
310-
default:
311-
return nil, errors.Errorf("unsupported cache type for cache %s: %s", cacheName, cacheBackend.Backend)
312-
}
313-
}
314-
315-
func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) {
273+
func createBucketCache(cacheName string, cacheBackend *BucketCacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) {
316274
if cacheBackend.Backend == "" {
317275
// No caching.
318276
return nil, nil
@@ -326,7 +284,7 @@ func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger
326284
for _, backend := range splitBackends {
327285
switch backend {
328286
case CacheBackendInMemory:
329-
inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, reg, cacheBackend.InMemory.toInMemoryChunkCacheConfig())
287+
inMemoryCache, err := cache.NewInMemoryCacheWithConfig(cacheName, logger, reg, cacheBackend.InMemory.toInMemoryCacheConfig())
330288
if err != nil {
331289
return nil, errors.Wrapf(err, "failed to create in-memory chunk cache")
332290
}
@@ -347,7 +305,7 @@ func createChunkCache(cacheName string, cacheBackend *ChunkCacheBackend, logger
347305
}
348306
}
349307

350-
return newMultiLevelChunkCache(cacheName, cacheBackend.MultiLevel, reg, caches...), nil
308+
return newMultiLevelBucketCache(cacheName, cacheBackend.MultiLevel, reg, caches...), nil
351309
}
352310

353311
type Matchers struct {

‎pkg/storage/tsdb/caching_bucket_test.go

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,82 +8,82 @@ import (
88
"github.com/stretchr/testify/assert"
99
)
1010

11-
func Test_ChunkCacheBackendValidation(t *testing.T) {
11+
func Test_BucketCacheBackendValidation(t *testing.T) {
1212
tests := map[string]struct {
13-
cfg ChunkCacheBackend
13+
cfg BucketCacheBackend
1414
expectedErr error
1515
}{
16-
"valid chunk cache type ('')": {
17-
cfg: ChunkCacheBackend{
16+
"valid bucket cache type ('')": {
17+
cfg: BucketCacheBackend{
1818
Backend: "",
1919
},
2020
expectedErr: nil,
2121
},
22-
"valid chunk cache type (in-memory)": {
23-
cfg: ChunkCacheBackend{
22+
"valid bucket cache type (in-memory)": {
23+
cfg: BucketCacheBackend{
2424
Backend: CacheBackendInMemory,
2525
},
2626
expectedErr: nil,
2727
},
28-
"valid chunk cache type (memcached)": {
29-
cfg: ChunkCacheBackend{
28+
"valid bucket cache type (memcached)": {
29+
cfg: BucketCacheBackend{
3030
Backend: CacheBackendMemcached,
3131
Memcached: MemcachedClientConfig{
3232
Addresses: "dns+localhost:11211",
3333
},
3434
},
3535
expectedErr: nil,
3636
},
37-
"valid chunk cache type (redis)": {
38-
cfg: ChunkCacheBackend{
37+
"valid bucket cache type (redis)": {
38+
cfg: BucketCacheBackend{
3939
Backend: CacheBackendRedis,
4040
Redis: RedisClientConfig{
4141
Addresses: "localhost:6379",
4242
},
4343
},
4444
expectedErr: nil,
4545
},
46-
"invalid chunk cache type": {
47-
cfg: ChunkCacheBackend{
46+
"invalid bucket cache type": {
47+
cfg: BucketCacheBackend{
4848
Backend: "dummy",
4949
},
50-
expectedErr: errUnsupportedChunkCacheBackend,
50+
expectedErr: errUnsupportedBucketCacheBackend,
5151
},
52-
"valid multi chunk cache type": {
53-
cfg: ChunkCacheBackend{
52+
"valid multi bucket cache type": {
53+
cfg: BucketCacheBackend{
5454
Backend: fmt.Sprintf("%s,%s,%s", CacheBackendInMemory, CacheBackendMemcached, CacheBackendRedis),
5555
Memcached: MemcachedClientConfig{
5656
Addresses: "dns+localhost:11211",
5757
},
5858
Redis: RedisClientConfig{
5959
Addresses: "localhost:6379",
6060
},
61-
MultiLevel: MultiLevelChunkCacheConfig{
61+
MultiLevel: MultiLevelBucketCacheConfig{
6262
MaxAsyncConcurrency: 1,
6363
MaxAsyncBufferSize: 1,
6464
MaxBackfillItems: 1,
6565
},
6666
},
6767
expectedErr: nil,
6868
},
69-
"duplicate multi chunk cache type": {
70-
cfg: ChunkCacheBackend{
69+
"duplicate multi bucket cache type": {
70+
cfg: BucketCacheBackend{
7171
Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendInMemory),
72-
MultiLevel: MultiLevelChunkCacheConfig{
72+
MultiLevel: MultiLevelBucketCacheConfig{
7373
MaxAsyncConcurrency: 1,
7474
MaxAsyncBufferSize: 1,
7575
MaxBackfillItems: 1,
7676
},
7777
},
78-
expectedErr: errDuplicatedChunkCacheBackend,
78+
expectedErr: errDuplicatedBucketCacheBackend,
7979
},
8080
"invalid max async concurrency": {
81-
cfg: ChunkCacheBackend{
81+
cfg: BucketCacheBackend{
8282
Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached),
8383
Memcached: MemcachedClientConfig{
8484
Addresses: "dns+localhost:11211",
8585
},
86-
MultiLevel: MultiLevelChunkCacheConfig{
86+
MultiLevel: MultiLevelBucketCacheConfig{
8787
MaxAsyncConcurrency: 0,
8888
MaxAsyncBufferSize: 1,
8989
MaxBackfillItems: 1,
@@ -92,12 +92,12 @@ func Test_ChunkCacheBackendValidation(t *testing.T) {
9292
expectedErr: errInvalidMaxAsyncConcurrency,
9393
},
9494
"invalid max async buffer size": {
95-
cfg: ChunkCacheBackend{
95+
cfg: BucketCacheBackend{
9696
Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached),
9797
Memcached: MemcachedClientConfig{
9898
Addresses: "dns+localhost:11211",
9999
},
100-
MultiLevel: MultiLevelChunkCacheConfig{
100+
MultiLevel: MultiLevelBucketCacheConfig{
101101
MaxAsyncConcurrency: 1,
102102
MaxAsyncBufferSize: 0,
103103
MaxBackfillItems: 1,
@@ -106,12 +106,12 @@ func Test_ChunkCacheBackendValidation(t *testing.T) {
106106
expectedErr: errInvalidMaxAsyncBufferSize,
107107
},
108108
"invalid max back fill items": {
109-
cfg: ChunkCacheBackend{
109+
cfg: BucketCacheBackend{
110110
Backend: fmt.Sprintf("%s,%s", CacheBackendInMemory, CacheBackendMemcached),
111111
Memcached: MemcachedClientConfig{
112112
Addresses: "dns+localhost:11211",
113113
},
114-
MultiLevel: MultiLevelChunkCacheConfig{
114+
MultiLevel: MultiLevelBucketCacheConfig{
115115
MaxAsyncConcurrency: 1,
116116
MaxAsyncBufferSize: 1,
117117
MaxBackfillItems: 0,

0 commit comments

Comments
 (0)