Skip to content

Commit 087f53f

Browse files
committed
support deploying tso service
Signed-off-by: Ryan Leung <[email protected]>
1 parent 38b6c96 commit 087f53f

File tree

21 files changed

+614
-10
lines changed

21 files changed

+614
-10
lines changed

components/cluster/command/upgrade.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222
func newUpgradeCmd() *cobra.Command {
2323
offlineMode := false
2424
ignoreVersionCheck := false
25-
var tidbVer, tikvVer, pdVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string
25+
var tidbVer, tikvVer, pdVer, tsoVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string
2626

2727
cmd := &cobra.Command{
2828
Use: "upgrade <cluster-name> <version>",
@@ -47,6 +47,7 @@ func newUpgradeCmd() *cobra.Command {
4747
spec.ComponentTiDB: tidbVer,
4848
spec.ComponentTiKV: tikvVer,
4949
spec.ComponentPD: pdVer,
50+
spec.ComponentTSO: tsoVer,
5051
spec.ComponentTiFlash: tiflashVer,
5152
spec.ComponentTiKVCDC: kvcdcVer,
5253
spec.ComponentCDC: cdcVer,
@@ -76,7 +77,8 @@ func newUpgradeCmd() *cobra.Command {
7677

7778
// cmd.Flags().StringVar(&tidbVer, "tidb-version", "", "Fix the version of tidb and no longer follows the cluster version.")
7879
cmd.Flags().StringVar(&tikvVer, "tikv-version", "", "Fix the version of tikv and no longer follows the cluster version.")
79-
cmd.Flags().StringVar(&pdVer, "pd-version", "", "Fix the version of pv and no longer follows the cluster version.")
80+
cmd.Flags().StringVar(&pdVer, "pd-version", "", "Fix the version of pd and no longer follows the cluster version.")
81+
cmd.Flags().StringVar(&tsoVer, "tso-version", "", "Fix the version of tso and no longer follows the cluster version.")
8082
cmd.Flags().StringVar(&tiflashVer, "tiflash-version", "", "Fix the version of tiflash and no longer follows the cluster version.")
8183
cmd.Flags().StringVar(&dashboardVer, "tidb-dashboard-version", "", "Fix the version of tidb-dashboard and no longer follows the cluster version.")
8284
cmd.Flags().StringVar(&cdcVer, "cdc-version", "", "Fix the version of cdc and no longer follows the cluster version.")

doc/rfcs/0001-separate-component-version-in-cluster.md

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ ComponentVersions struct {
2323
TiKV string `yaml:"tikv,omitempty"`
2424
TiFlash string `yaml:"tiflash,omitempty"`
2525
PD string `yaml:"pd,omitempty"`
26+
TSO string `yaml:"tso,omitempty"`
2627
Dashboard string `yaml:"tidb_dashboard,omitempty"`
2728
Pump string `yaml:"pump,omitempty"`
2829
Drainer string `yaml:"drainer,omitempty"`

embed/templates/config/prometheus.yml.tpl

+15
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,21 @@ scrape_configs:
155155
- targets:
156156
{{- range .PDAddrs}}
157157
- '{{.}}'
158+
{{- end}}
159+
- job_name: "tso"
160+
honor_labels: true # don't overwrite job & instance labels
161+
{{- if .TLSEnabled}}
162+
scheme: https
163+
tls_config:
164+
insecure_skip_verify: false
165+
ca_file: ../tls/ca.crt
166+
cert_file: ../tls/prometheus.crt
167+
key_file: ../tls/prometheus.pem
168+
{{- end}}
169+
static_configs:
170+
- targets:
171+
{{- range .TSOAddrs}}
172+
- '{{.}}'
158173
{{- end}}
159174
{{- if .TiFlashStatusAddrs}}
160175
- job_name: "tiflash"

embed/templates/scripts/run_pd.sh.tpl

+7-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,15 @@ DEPLOY_DIR={{.DeployDir}}
77

88
cd "${DEPLOY_DIR}" || exit 1
99

10+
exec \
1011
{{- if .NumaNode}}
11-
exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server \
12-
{{- else}}
13-
exec env GODEBUG=madvdontneed=1 bin/pd-server \
12+
numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} \
1413
{{- end}}
14+
env GODEBUG=madvdontneed=1 \
15+
{{- if .MSMode}}
16+
PD_SERVICE_MODE=api \
17+
{{- end}}
18+
bin/pd-server \
1519
--name="{{.Name}}" \
1620
--client-urls="{{.ClientURL}}" \
1721
--advertise-client-urls="{{.AdvertiseClientURL}}" \

embed/templates/scripts/run_pd_scale.sh.tpl

+7-3
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,15 @@ DEPLOY_DIR={{.DeployDir}}
77

88
cd "${DEPLOY_DIR}" || exit 1
99

10+
exec \
1011
{{- if .NumaNode}}
11-
exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server \
12-
{{- else}}
13-
exec env GODEBUG=madvdontneed=1 bin/pd-server \
12+
numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} \
1413
{{- end}}
14+
env GODEBUG=madvdontneed=1 \
15+
{{- if .MSMode}}
16+
PD_SERVICE_MODE=api \
17+
{{- end}}
18+
bin/pd-server \
1519
--name="{{.Name}}" \
1620
--client-urls="{{.ClientURL}}" \
1721
--advertise-client-urls="{{.AdvertiseClientURL}}" \
+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# WARNING: This file was auto-generated. Do not edit!
5+
# All your edit might be overwritten!
6+
DEPLOY_DIR={{.DeployDir}}
7+
8+
cd "${DEPLOY_DIR}" || exit 1
9+
10+
{{- if .NumaNode}}
11+
exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} env GODEBUG=madvdontneed=1 bin/pd-server services tso\
12+
{{- else}}
13+
exec env GODEBUG=madvdontneed=1 bin/pd-server services tso \
14+
{{- end}}
15+
--backend-endpoints="{{.BackendEndpoints}}" \
16+
--listen-addr="{{.ListenURL}}" \
17+
--advertise-listen-addr="{{.AdvertiseListenURL}}" \
18+
--config=conf/tso.toml \
19+
--log-file="{{.LogDir}}/tso.log" 2>> "{{.LogDir}}/tso_stderr.log"

pkg/cluster/ansible/import_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ server_configs:
124124
binlog.enable: true
125125
tikv: {}
126126
pd: {}
127+
tso: {}
127128
tidb_dashboard: {}
128129
tiflash: {}
129130
tiproxy: {}
@@ -138,6 +139,7 @@ tikv_servers: []
138139
tiflash_servers: []
139140
tiproxy_servers: []
140141
pd_servers: []
142+
tso_servers: []
141143
monitoring_servers: []
142144
`)
143145

pkg/cluster/ansible/test-data/meta.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ topology:
1919
binlog.enable: true
2020
tikv: {}
2121
pd: {}
22+
tso: {}
2223
tidb_dashboard: {}
2324
tiflash: {}
2425
tiproxy: {}
@@ -195,3 +196,4 @@ topology:
195196
arch: amd64
196197
os: linux
197198
tiproxy_servers: []
199+
tso_servers: []

pkg/cluster/api/pdapi.go

+116
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ var (
139139
pdStoresURI = "pd/api/v1/stores"
140140
pdStoresLimitURI = "pd/api/v1/stores/limit"
141141
pdRegionsCheckURI = "pd/api/v1/regions/check"
142+
pdServicePrimaryURI = "pd/api/v2/ms/primary"
142143
)
143144

144145
func tryURLs(endpoints []string, f func(endpoint string) ([]byte, error)) ([]byte, error) {
@@ -978,3 +979,118 @@ func (pc *PDClient) SetAllStoreLimits(value int) error {
978979
pc.l().Debugf("setting store limit: %d", value)
979980
return pc.updateConfig(pdStoresLimitURI, bytes.NewBuffer(body))
980981
}
982+
983+
// GetServicePrimary queries for the primary of a service
984+
func (pc *PDClient) GetServicePrimary(service string) (string, error) {
985+
endpoints := pc.getEndpoints(fmt.Sprintf("%s/%s", pdServicePrimaryURI, service))
986+
987+
var primary string
988+
_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
989+
body, err := pc.httpClient.Get(pc.ctx, endpoint)
990+
if err != nil {
991+
return body, err
992+
}
993+
994+
return body, json.Unmarshal(body, &primary)
995+
})
996+
return primary, err
997+
}
998+
999+
const (
1000+
tsoStatusURI = "status"
1001+
)
1002+
1003+
// TSOClient is an HTTP client of the TSO server
1004+
type TSOClient struct {
1005+
version string
1006+
addrs []string
1007+
tlsEnabled bool
1008+
httpClient *utils.HTTPClient
1009+
ctx context.Context
1010+
}
1011+
1012+
// NewTSOClient returns a new TSOClient, the context must have
1013+
// a *logprinter.Logger as value of "logger"
1014+
func NewTSOClient(
1015+
ctx context.Context,
1016+
addrs []string,
1017+
timeout time.Duration,
1018+
tlsConfig *tls.Config,
1019+
) *TSOClient {
1020+
enableTLS := false
1021+
if tlsConfig != nil {
1022+
enableTLS = true
1023+
}
1024+
1025+
if _, ok := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger); !ok {
1026+
panic("the context must have logger inside")
1027+
}
1028+
1029+
cli := &TSOClient{
1030+
addrs: addrs,
1031+
tlsEnabled: enableTLS,
1032+
httpClient: utils.NewHTTPClient(timeout, tlsConfig),
1033+
ctx: ctx,
1034+
}
1035+
1036+
cli.tryIdentifyVersion()
1037+
return cli
1038+
}
1039+
1040+
// func (tc *TSOClient) l() *logprinter.Logger {
1041+
// return tc.ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
1042+
// }
1043+
1044+
func (tc *TSOClient) tryIdentifyVersion() {
1045+
endpoints := tc.getEndpoints(tsoStatusURI)
1046+
response := map[string]string{}
1047+
_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
1048+
body, err := tc.httpClient.Get(tc.ctx, endpoint)
1049+
if err != nil {
1050+
return body, err
1051+
}
1052+
1053+
return body, json.Unmarshal(body, &response)
1054+
})
1055+
if err == nil {
1056+
tc.version = response["version"]
1057+
}
1058+
}
1059+
1060+
// GetURL builds the client URL of PDClient
1061+
func (tc *TSOClient) GetURL(addr string) string {
1062+
httpPrefix := "http"
1063+
if tc.tlsEnabled {
1064+
httpPrefix = "https"
1065+
}
1066+
return fmt.Sprintf("%s://%s", httpPrefix, addr)
1067+
}
1068+
1069+
func (tc *TSOClient) getEndpoints(uri string) (endpoints []string) {
1070+
for _, addr := range tc.addrs {
1071+
endpoint := fmt.Sprintf("%s/%s", tc.GetURL(addr), uri)
1072+
endpoints = append(endpoints, endpoint)
1073+
}
1074+
1075+
return
1076+
}
1077+
1078+
// CheckHealth checks the health of TSO node.
1079+
func (tc *TSOClient) CheckHealth() error {
1080+
endpoints := tc.getEndpoints(tsoStatusURI)
1081+
1082+
_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
1083+
body, err := tc.httpClient.Get(tc.ctx, endpoint)
1084+
if err != nil {
1085+
return body, err
1086+
}
1087+
1088+
return body, nil
1089+
})
1090+
1091+
if err != nil {
1092+
return err
1093+
}
1094+
1095+
return nil
1096+
}

pkg/cluster/clusterutil/cluster.go

+3
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ func (r *repositoryT) VerifyComponent(comp, version, target string) error {
8484
}
8585

8686
func (r *repositoryT) ComponentBinEntry(comp, version string) (string, error) {
87+
if comp == "tso" {
88+
comp = "pd"
89+
}
8790
versionItem, err := r.repo.ComponentVersion(comp, version, true)
8891
if err != nil {
8992
return "", err

pkg/cluster/executor/ssh.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ func (e *EasySSHExecutor) Execute(ctx context.Context, cmd string, sudo bool, ti
144144

145145
// set a basic PATH in case it's empty on login
146146
cmd = fmt.Sprintf("PATH=$PATH:/bin:/sbin:/usr/bin:/usr/sbin; %s", cmd)
147-
147+
148148
if e.Locale != "" {
149149
cmd = fmt.Sprintf("export LANG=%s; %s", e.Locale, cmd)
150150
}

pkg/cluster/spec/instance.go

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const (
4040
ComponentTiDB = "tidb"
4141
ComponentTiKV = "tikv"
4242
ComponentPD = "pd"
43+
ComponentTSO = "tso"
4344
ComponentTiFlash = "tiflash"
4445
ComponentTiProxy = "tiproxy"
4546
ComponentGrafana = "grafana"

pkg/cluster/spec/monitoring.go

+7
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,13 @@ func (i *MonitorInstance) InitConfig(
261261
cfig.AddPD(pd.Host, uint64(pd.ClientPort))
262262
}
263263
}
264+
if servers, found := topoHasField("TSOServers"); found {
265+
for i := 0; i < servers.Len(); i++ {
266+
tso := servers.Index(i).Interface().(*TSOSpec)
267+
uniqueHosts.Insert(tso.Host)
268+
cfig.AddTSO(tso.Host, uint64(tso.Port))
269+
}
270+
}
264271
if servers, found := topoHasField("TiKVServers"); found {
265272
for i := 0; i < servers.Len(); i++ {
266273
kv := servers.Index(i).Interface().(*TiKVSpec)

pkg/cluster/spec/pd.go

+3
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type PDSpec struct {
5050
LogDir string `yaml:"log_dir,omitempty"`
5151
Source string `yaml:"source,omitempty" validate:"source:editable"`
5252
NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"`
53+
MSMode bool `yaml:"ms_mode,omitempty"`
5354
Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"`
5455
ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
5556
Arch string `yaml:"arch,omitempty"`
@@ -254,6 +255,7 @@ func (i *PDInstance) InitConfig(
254255
LogDir: paths.Log,
255256
InitialCluster: strings.Join(initialCluster, ","),
256257
NumaNode: spec.NumaNode,
258+
MSMode: spec.MSMode,
257259
}
258260

259261
fp := filepath.Join(paths.Cache, fmt.Sprintf("run_pd_%s_%d.sh", i.GetHost(), i.GetPort()))
@@ -378,6 +380,7 @@ func (i *PDInstance) ScaleConfig(
378380
LogDir: paths.Log,
379381
InitialCluster: strings.Join(initialCluster, ","),
380382
NumaNode: spec.NumaNode,
383+
MSMode: spec.MSMode,
381384
}
382385

383386
join := []string{}

0 commit comments

Comments
 (0)