Skip to content

Commit b9aa9de

Browse files
breezewishbreezewish
authored
playground: Add tidb-cse mode (#2386)
1 parent aa448ad commit b9aa9de

File tree

12 files changed

+170
-98
lines changed

12 files changed

+170
-98
lines changed

‎components/playground/instance/instance.go

+8
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ type Config struct {
3737
Version string `yaml:"version"`
3838
}
3939

40+
// CSEOptions contains configs to run TiDB cluster in CSE mode.
41+
type CSEOptions struct {
42+
S3Endpoint string `yaml:"s3_endpoint"`
43+
Bucket string `yaml:"bucket"`
44+
AccessKey string `yaml:"access_key"`
45+
SecretKey string `yaml:"secret_key"`
46+
}
47+
4048
type instance struct {
4149
ID int
4250
Dir string

‎components/playground/instance/pd.go

+8-10
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,11 @@ type PDInstance struct {
4646
joinEndpoints []*PDInstance
4747
pds []*PDInstance
4848
Process
49+
isCSEMode bool
4950
}
5051

5152
// NewPDInstance return a PDInstance
52-
func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int) *PDInstance {
53+
func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int, isCSEMode bool) *PDInstance {
5354
if port <= 0 {
5455
port = 2379
5556
}
@@ -63,8 +64,9 @@ func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, p
6364
StatusPort: utils.MustGetFreePort(host, port),
6465
ConfigPath: configPath,
6566
},
66-
Role: role,
67-
pds: pds,
67+
Role: role,
68+
pds: pds,
69+
isCSEMode: isCSEMode,
6870
}
6971
}
7072

@@ -112,8 +114,8 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
112114
fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
113115
fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
114116
fmt.Sprintf("--log-file=%s", inst.LogFile()),
117+
fmt.Sprintf("--config=%s", configPath),
115118
}...)
116-
117119
switch {
118120
case len(inst.initEndpoints) > 0:
119121
endpoints := make([]string, 0)
@@ -140,9 +142,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
140142
fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
141143
fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
142144
fmt.Sprintf("--log-file=%s", inst.LogFile()),
143-
}
144-
if inst.ConfigPath != "" {
145-
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
145+
fmt.Sprintf("--config=%s", configPath),
146146
}
147147
case PDRoleScheduling:
148148
endpoints := pdEndpoints(inst.pds, true)
@@ -153,9 +153,7 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
153153
fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
154154
fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")),
155155
fmt.Sprintf("--log-file=%s", inst.LogFile()),
156-
}
157-
if inst.ConfigPath != "" {
158-
args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
156+
fmt.Sprintf("--config=%s", configPath),
159157
}
160158
}
161159

‎components/playground/instance/pd_config.go

+10
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,15 @@ package instance
1616
func (inst *PDInstance) getConfig() map[string]any {
1717
config := make(map[string]any)
1818
config["schedule.patrol-region-interval"] = "100ms"
19+
20+
if inst.isCSEMode {
21+
config["keyspace.pre-alloc"] = []string{"mykeyspace"}
22+
config["replication.enable-placement-rules"] = true
23+
config["replication.max-replica"] = 1
24+
config["schedule.merge-schedule-limit"] = 0
25+
config["schedule.low-space-ration"] = 1.0
26+
config["schedule.replica-schedule-limit"] = 500
27+
}
28+
1929
return config
2030
}

‎components/playground/instance/tidb.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ type TiDBInstance struct {
3131
Process
3232
tiproxyCertDir string
3333
enableBinlog bool
34-
isDisaggMode bool
34+
isCSEMode bool
3535
}
3636

3737
// NewTiDBInstance return a TiDBInstance
38-
func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isDisaggMode bool) *TiDBInstance {
38+
func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, tiproxyCertDir string, enableBinlog bool, isCSEMode bool) *TiDBInstance {
3939
if port <= 0 {
4040
port = 4000
4141
}
@@ -52,7 +52,7 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int,
5252
tiproxyCertDir: tiproxyCertDir,
5353
pds: pds,
5454
enableBinlog: enableBinlog,
55-
isDisaggMode: isDisaggMode,
55+
isCSEMode: isCSEMode,
5656
}
5757
}
5858

‎components/playground/instance/tidb_config.go

+28-1
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,36 @@ func (inst *TiDBInstance) getConfig() map[string]any {
2222
config := make(map[string]any)
2323
config["security.auto-tls"] = true
2424

25-
if inst.isDisaggMode {
25+
if inst.isCSEMode {
26+
config["keyspace-name"] = "mykeyspace"
27+
config["enable-safe-point-v2"] = true
28+
config["force-enable-vector-type"] = true
2629
config["use-autoscaler"] = false
2730
config["disaggregated-tiflash"] = true
31+
config["ratelimit.full-speed"] = 1048576000
32+
config["ratelimit.full-speed-capacity"] = 1048576000
33+
config["ratelimit.low-speed-watermark"] = 1048576000000
34+
config["ratelimit.block-write-watermark"] = 1048576000000
35+
config["security.enable-sem"] = false
36+
config["tiflash-replicas.constraints"] = []any{
37+
map[string]any{
38+
"key": "engine",
39+
"op": "in",
40+
"values": []string{
41+
"tiflash",
42+
},
43+
},
44+
map[string]any{
45+
"key": "engine_role",
46+
"op": "in",
47+
"values": []string{
48+
"write",
49+
},
50+
},
51+
}
52+
config["tiflash-replicas.group-id"] = "enable_s3_wn_region"
53+
config["tiflash-replicas.extra-s3-rule"] = false
54+
config["tiflash-replicas.min-count"] = 1
2855
}
2956

3057
tiproxyCrtPath := filepath.Join(inst.tiproxyCertDir, "tiproxy.crt")

‎components/playground/instance/tiflash.go

+3-11
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,11 @@ const (
3939
TiFlashRoleDisaggCompute TiFlashRole = "compute"
4040
)
4141

42-
// DisaggOptions contains configs to run TiFlash in disaggregated mode.
43-
type DisaggOptions struct {
44-
S3Endpoint string `yaml:"s3_endpoint"`
45-
Bucket string `yaml:"bucket"`
46-
AccessKey string `yaml:"access_key"`
47-
SecretKey string `yaml:"secret_key"`
48-
}
49-
5042
// TiFlashInstance represent a running TiFlash
5143
type TiFlashInstance struct {
5244
instance
5345
Role TiFlashRole
54-
DisaggOpts DisaggOptions
46+
cseOpts CSEOptions
5547
TCPPort int
5648
ServicePort int
5749
ProxyPort int
@@ -62,7 +54,7 @@ type TiFlashInstance struct {
6254
}
6355

6456
// NewTiFlashInstance return a TiFlashInstance
65-
func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
57+
func NewTiFlashInstance(role TiFlashRole, cseOptions CSEOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance {
6658
if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute {
6759
panic(fmt.Sprintf("Unknown TiFlash role %s", role))
6860
}
@@ -82,7 +74,7 @@ func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath,
8274
ConfigPath: configPath,
8375
},
8476
Role: role,
85-
DisaggOpts: disaggOptions,
77+
cseOpts: cseOptions,
8678
TCPPort: utils.MustGetFreePort(host, 9100), // 9000 for default object store port
8779
ServicePort: utils.MustGetFreePort(host, 3930),
8880
ProxyPort: utils.MustGetFreePort(host, 20170),

‎components/playground/instance/tiflash_config.go

+28-13
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,18 @@ func (inst *TiFlashInstance) getProxyConfig() map[string]any {
2121
config["raftdb.max-open-files"] = 256
2222
config["storage.reserve-space"] = 0
2323
config["storage.reserve-raft-space"] = 0
24+
25+
if inst.Role == TiFlashRoleDisaggWrite {
26+
config["storage.api-version"] = 2
27+
config["storage.enable-ttl"] = true
28+
config["dfs.prefix"] = "tikv"
29+
config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
30+
config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
31+
config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
32+
config["dfs.s3-bucket"] = inst.cseOpts.Bucket
33+
config["dfs.s3-region"] = "local"
34+
}
35+
2436
return config
2537
}
2638

@@ -31,23 +43,26 @@ func (inst *TiFlashInstance) getConfig() map[string]any {
3143
config["logger.level"] = "debug"
3244

3345
if inst.Role == TiFlashRoleDisaggWrite {
34-
config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint
35-
config["storage.s3.bucket"] = inst.DisaggOpts.Bucket
36-
config["storage.s3.root"] = "/"
37-
config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey
38-
config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey
46+
config["enable_safe_point_v2"] = true
47+
config["storage.api_version"] = 2
48+
config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
49+
config["storage.s3.bucket"] = inst.cseOpts.Bucket
50+
config["storage.s3.root"] = "/tiflash-cse/"
51+
config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
52+
config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
53+
config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
3954
config["flash.disaggregated_mode"] = "tiflash_write"
40-
config["flash.use_autoscaler"] = false
4155
} else if inst.Role == TiFlashRoleDisaggCompute {
42-
config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint
43-
config["storage.s3.bucket"] = inst.DisaggOpts.Bucket
44-
config["storage.s3.root"] = "/"
45-
config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey
46-
config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey
56+
config["enable_safe_point_v2"] = true
57+
config["storage.s3.endpoint"] = inst.cseOpts.S3Endpoint
58+
config["storage.s3.bucket"] = inst.cseOpts.Bucket
59+
config["storage.s3.root"] = "/tiflash-cse/"
60+
config["storage.s3.access_key_id"] = inst.cseOpts.AccessKey
61+
config["storage.s3.secret_access_key"] = inst.cseOpts.SecretKey
4762
config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache")
48-
config["storage.remote.cache.capacity"] = 1000000000 // 1GB
63+
config["storage.remote.cache.capacity"] = 20000000000 // 20GB
64+
config["storage.main.dir"] = []string{filepath.Join(inst.Dir, "main_data")}
4965
config["flash.disaggregated_mode"] = "tiflash_compute"
50-
config["flash.use_autoscaler"] = false
5166
}
5267

5368
return config

‎components/playground/instance/tikv.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@ type TiKVInstance struct {
2828
instance
2929
pds []*PDInstance
3030
Process
31+
isCSEMode bool
32+
cseOpts CSEOptions
3133
}
3234

3335
// NewTiKVInstance return a TiKVInstance
34-
func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiKVInstance {
36+
func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance, isCSEMode bool, cseOptions CSEOptions) *TiKVInstance {
3537
if port <= 0 {
3638
port = 20160
3739
}
@@ -45,7 +47,9 @@ func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port
4547
StatusPort: utils.MustGetFreePort(host, 20180),
4648
ConfigPath: configPath,
4749
},
48-
pds: pds,
50+
pds: pds,
51+
isCSEMode: isCSEMode,
52+
cseOpts: cseOptions,
4953
}
5054
}
5155

‎components/playground/instance/tikv_config.go

+12
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,17 @@ func (inst *TiKVInstance) getConfig() map[string]any {
1919
config["raftdb.max-open-files"] = 256
2020
config["storage.reserve-space"] = 0
2121
config["storage.reserve-raft-space"] = 0
22+
23+
if inst.isCSEMode {
24+
config["storage.api-version"] = 2
25+
config["storage.enable-ttl"] = true
26+
config["dfs.prefix"] = "tikv"
27+
config["dfs.s3-endpoint"] = inst.cseOpts.S3Endpoint
28+
config["dfs.s3-key-id"] = inst.cseOpts.AccessKey
29+
config["dfs.s3-secret-key"] = inst.cseOpts.SecretKey
30+
config["dfs.s3-bucket"] = inst.cseOpts.Bucket
31+
config["dfs.s3-region"] = "local"
32+
}
33+
2234
return config
2335
}

0 commit comments

Comments
 (0)