Skip to content

Commit 8b4630a

Browse files
SungJin1212SungJin1212
authored
Enable compactor and alertmanager in target all (#6204)
1 parent d829d65 commit 8b4630a

13 files changed

+176
-61
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
## master / unreleased
44

5+
* [CHANGE] Enable Compactor and Alertmanager in target all. #6204
56
* [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151
67
* [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
78
* [ENHANCEMENT] Query Frontend: Add info field to query response. #6207

‎integration/api_endpoints_test.go

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"io"
99
"net/http"
10+
"path/filepath"
1011
"testing"
1112

1213
"github.com/stretchr/testify/assert"
@@ -22,10 +23,19 @@ func TestIndexAPIEndpoint(t *testing.T) {
2223
require.NoError(t, err)
2324
defer s.Close()
2425

26+
configOverrides := map[string]string{
27+
// alert manager
28+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
29+
"-alertmanager-storage.backend": "local",
30+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
31+
}
32+
// make alert manager config dir
33+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
34+
2535
// Start Cortex in single binary mode, reading the config from file.
2636
require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
2737

28-
cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
38+
cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
2939
require.NoError(t, s.StartAndWaitReady(cortex1))
3040

3141
// GET / should succeed
@@ -44,10 +54,19 @@ func TestConfigAPIEndpoint(t *testing.T) {
4454
require.NoError(t, err)
4555
defer s.Close()
4656

57+
configOverrides := map[string]string{
58+
// alert manager
59+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
60+
"-alertmanager-storage.backend": "local",
61+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
62+
}
63+
// make alert manager config dir
64+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
65+
4766
// Start Cortex in single binary mode, reading the config from file.
4867
require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
4968

50-
cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
69+
cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
5170
require.NoError(t, s.StartAndWaitReady(cortex1))
5271

5372
// Get config from /config API endpoint.
@@ -62,6 +81,7 @@ func TestConfigAPIEndpoint(t *testing.T) {
6281
// Start again Cortex in single binary with the exported config
6382
// and ensure it starts (pass the readiness probe).
6483
require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body))
65-
cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, nil, "", 9009, 9095)
84+
configOverrides["-alertmanager.cluster.peers"] = cortex1.HTTPEndpoint()
85+
cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, configOverrides, "", 9009, 9095)
6686
require.NoError(t, s.StartAndWaitReady(cortex2))
6787
}

‎integration/getting_started_single_process_config_local_test.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package integration
55

66
import (
7+
"path/filepath"
78
"testing"
89
"time"
910

@@ -24,7 +25,14 @@ func TestGettingStartedSingleProcessConfigWithFilesystem(t *testing.T) {
2425
// Start Cortex components.
2526
require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
2627

27-
flags := map[string]string{}
28+
flags := map[string]string{
29+
// alert manager
30+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
31+
"-alertmanager-storage.backend": "local",
32+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
33+
}
34+
// make alert manager config dir
35+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
2836

2937
cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
3038
require.NoError(t, s.StartAndWaitReady(cortex))

‎integration/getting_started_single_process_config_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package integration
55

66
import (
77
"fmt"
8+
"path/filepath"
89
"testing"
910
"time"
1011

@@ -38,7 +39,13 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) {
3839
"-blocks-storage.s3.bucket-name": bucketName,
3940
"-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName),
4041
"-blocks-storage.s3.insecure": "true",
42+
// alert manager
43+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
44+
"-alertmanager-storage.backend": "local",
45+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
4146
}
47+
// make alert manager config dir
48+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
4249

4350
cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
4451
require.NoError(t, s.StartAndWaitReady(cortex))

‎integration/getting_started_with_gossiped_ring_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package integration
55

66
import (
77
"fmt"
8+
"path/filepath"
89
"testing"
910
"time"
1011

@@ -45,8 +46,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) {
4546
"-blocks-storage.s3.insecure": "true",
4647
"-store-gateway.sharding-ring.wait-stability-min-duration": "0", // start quickly
4748
"-store-gateway.sharding-ring.wait-stability-max-duration": "0", // start quickly
49+
// alert manager
50+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
51+
"-alertmanager-storage.backend": "local",
52+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
4853
}
4954

55+
// make alert manager config dir
56+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
57+
5058
// This cortex will fail to join the cluster configured in yaml file. That's fine.
5159
cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", "config1.yaml", e2e.MergeFlags(flags, map[string]string{
5260
"-ingester.lifecycler.addr": networkName + "-cortex-1", // Ingester's hostname in docker setup

‎integration/integration_memberlist_single_binary_test.go

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
4343
require.NoError(t, err)
4444
defer s.Close()
4545

46+
// make alert manager config dir
47+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
48+
4649
// Start dependencies
4750
minio := e2edb.NewMinio(9000, bucketName)
4851
// Look ma, no Consul!
@@ -116,16 +119,21 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
116119
}
117120

118121
func newSingleBinary(name string, servername string, join string, testFlags map[string]string) *e2ecortex.CortexService {
119-
flags := map[string]string{
120-
"-ingester.final-sleep": "0s",
121-
"-ingester.join-after": "0s", // join quickly
122-
"-ingester.min-ready-duration": "0s",
123-
"-ingester.num-tokens": "512",
124-
"-ingester.observe-period": "5s", // to avoid conflicts in tokens
125-
"-ring.store": "memberlist",
126-
"-memberlist.bind-port": "8000",
127-
"-memberlist.left-ingesters-timeout": "600s", // effectively disable
128-
}
122+
flags := mergeFlags(
123+
AlertmanagerLocalFlags(),
124+
map[string]string{
125+
"-ingester.final-sleep": "0s",
126+
"-ingester.join-after": "0s", // join quickly
127+
"-ingester.min-ready-duration": "0s",
128+
"-ingester.num-tokens": "512",
129+
"-ingester.observe-period": "5s", // to avoid conflicts in tokens
130+
"-ring.store": "memberlist",
131+
"-memberlist.bind-port": "8000",
132+
"-memberlist.left-ingesters-timeout": "600s", // effectively disable
133+
// alert manager
134+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
135+
},
136+
)
129137

130138
if join != "" {
131139
flags["-memberlist.join"] = join
@@ -158,6 +166,9 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) {
158166
require.NoError(t, err)
159167
defer s.Close()
160168

169+
// make alert manager config dir
170+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
171+
161172
minio := e2edb.NewMinio(9000, bucketName)
162173
require.NoError(t, s.StartAndWaitReady(minio))
163174

‎integration/otlp_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package integration
66
import (
77
"fmt"
88
"math/rand"
9+
"path/filepath"
910
"testing"
1011
"time"
1112

@@ -41,7 +42,13 @@ func TestOTLP(t *testing.T) {
4142
"-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName),
4243
"-blocks-storage.s3.insecure": "true",
4344
"-blocks-storage.tsdb.enable-native-histograms": "true",
45+
// alert manager
46+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
47+
"-alertmanager-storage.backend": "local",
48+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
4449
}
50+
// make alert manager config dir
51+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
4552

4653
cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
4754
require.NoError(t, s.StartAndWaitReady(cortex))

‎integration/querier_test.go

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -365,26 +365,33 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
365365

366366
// Configure the blocks storage to frequently compact TSDB head
367367
// and ship blocks to the storage.
368-
flags := mergeFlags(BlocksStorageFlags(), map[string]string{
369-
"-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(),
370-
"-blocks-storage.tsdb.ship-interval": "1s",
371-
"-blocks-storage.bucket-store.sync-interval": "1s",
372-
"-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(),
373-
"-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend,
374-
"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
375-
"-querier.query-store-for-labels-enabled": "true",
376-
"-querier.thanos-engine": strconv.FormatBool(thanosEngine),
377-
// Ingester.
378-
"-ring.store": "consul",
379-
"-consul.hostname": consul.NetworkHTTPEndpoint(),
380-
// Distributor.
381-
"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
382-
// Store-gateway.
383-
"-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled),
384-
"-store-gateway.sharding-ring.store": "consul",
385-
"-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(),
386-
"-store-gateway.sharding-ring.replication-factor": "1",
387-
})
368+
flags := mergeFlags(
369+
BlocksStorageFlags(),
370+
AlertmanagerLocalFlags(),
371+
map[string]string{
372+
"-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(),
373+
"-blocks-storage.tsdb.ship-interval": "1s",
374+
"-blocks-storage.bucket-store.sync-interval": "1s",
375+
"-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(),
376+
"-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend,
377+
"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
378+
"-querier.query-store-for-labels-enabled": "true",
379+
"-querier.thanos-engine": strconv.FormatBool(thanosEngine),
380+
// Ingester.
381+
"-ring.store": "consul",
382+
"-consul.hostname": consul.NetworkHTTPEndpoint(),
383+
// Distributor.
384+
"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
385+
// Store-gateway.
386+
"-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled),
387+
"-store-gateway.sharding-ring.store": "consul",
388+
"-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(),
389+
"-store-gateway.sharding-ring.replication-factor": "1",
390+
// alert manager
391+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
392+
},
393+
)
394+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml)))
388395

389396
// Add the cache address to the flags.
390397
if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {

‎integration/query_fuzz_test.go

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,25 +45,32 @@ func TestVerticalShardingFuzz(t *testing.T) {
4545
consul2 := e2edb.NewConsulWithName("consul2")
4646
require.NoError(t, s.StartAndWaitReady(consul1, consul2))
4747

48-
flags := map[string]string{
49-
"-store.engine": blocksStorageEngine,
50-
"-blocks-storage.backend": "filesystem",
51-
"-blocks-storage.tsdb.head-compaction-interval": "4m",
52-
"-blocks-storage.tsdb.block-ranges-period": "2h",
53-
"-blocks-storage.tsdb.ship-interval": "1h",
54-
"-blocks-storage.bucket-store.sync-interval": "15m",
55-
"-blocks-storage.tsdb.retention-period": "2h",
56-
"-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory,
57-
"-blocks-storage.bucket-store.bucket-index.enabled": "true",
58-
"-querier.query-store-for-labels-enabled": "true",
59-
// Ingester.
60-
"-ring.store": "consul",
61-
"-consul.hostname": consul1.NetworkHTTPEndpoint(),
62-
// Distributor.
63-
"-distributor.replication-factor": "1",
64-
// Store-gateway.
65-
"-store-gateway.sharding-enabled": "false",
66-
}
48+
flags := mergeFlags(
49+
AlertmanagerLocalFlags(),
50+
map[string]string{
51+
"-store.engine": blocksStorageEngine,
52+
"-blocks-storage.backend": "filesystem",
53+
"-blocks-storage.tsdb.head-compaction-interval": "4m",
54+
"-blocks-storage.tsdb.block-ranges-period": "2h",
55+
"-blocks-storage.tsdb.ship-interval": "1h",
56+
"-blocks-storage.bucket-store.sync-interval": "15m",
57+
"-blocks-storage.tsdb.retention-period": "2h",
58+
"-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory,
59+
"-blocks-storage.bucket-store.bucket-index.enabled": "true",
60+
"-querier.query-store-for-labels-enabled": "true",
61+
// Ingester.
62+
"-ring.store": "consul",
63+
"-consul.hostname": consul1.NetworkHTTPEndpoint(),
64+
// Distributor.
65+
"-distributor.replication-factor": "1",
66+
// Store-gateway.
67+
"-store-gateway.sharding-enabled": "false",
68+
// alert manager
69+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
70+
},
71+
)
72+
// make alert manager config dir
73+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
6774

6875
path1 := path.Join(s.SharedDir(), "cortex-1")
6976
path2 := path.Join(s.SharedDir(), "cortex-2")

‎integration/ruler_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,13 @@ func TestRulerAPISingleBinary(t *testing.T) {
154154
"-ruler-storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"),
155155
"-ruler.poll-interval": "2s",
156156
"-ruler.rule-path": filepath.Join(e2e.ContainerSharedDir, "rule_tmp/"),
157+
// alert manager
158+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
159+
"-alertmanager-storage.backend": "local",
160+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
157161
}
162+
// make alert manager config dir
163+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
158164

159165
// Start Cortex components.
160166
require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))

‎integration/runtime_config_test.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,26 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) {
4040
name: "no storage backend provided",
4141
flags: map[string]string{
4242
"-runtime-config.file": filePath,
43+
// alert manager
44+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
45+
"-alertmanager-storage.backend": "local",
46+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
4347
},
4448
},
4549
{
4650
name: "filesystem as storage backend",
4751
flags: map[string]string{
4852
"-runtime-config.file": filePath,
4953
"-runtime-config.backend": "filesystem",
54+
// alert manager
55+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
56+
"-alertmanager-storage.backend": "local",
57+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
5058
},
5159
},
5260
}
61+
// make alert manager config dir
62+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
5363

5464
for i, tt := range tests {
5565
t.Run(tt.name, func(t *testing.T) {
@@ -79,7 +89,14 @@ func TestLoadRuntimeConfigFromCloudStorage(t *testing.T) {
7989
"-runtime-config.s3.insecure": "true",
8090
"-runtime-config.file": configFileName,
8191
"-runtime-config.reload-period": "2s",
92+
// alert manager
93+
"-alertmanager.web.external-url": "http://localhost/alertmanager",
94+
"-alertmanager-storage.backend": "local",
95+
"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
8296
}
97+
// make alert manager config dir
98+
require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
99+
83100
// create s3 storage backend
84101
minio := e2edb.NewMinio(9000, bucketName)
85102
require.NoError(t, s.StartAndWaitReady(minio))

0 commit comments

Comments
 (0)