Skip to content

Commit 00de493

Browse files
acekingkeacekingke
committed
*: support juicefs #745 
1 parent 41b6f3c commit 00de493

26 files changed

+1117
-10
lines changed

‎Dockerfile.sidecar

+10-3
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,20 @@ RUN set -ex; \
4343
ARG XTRABACKUP_PKG=percona-xtrabackup-24
4444
RUN set -ex; \
4545
apt-get update; \
46-
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
46+
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \
4747
wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
4848
dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
4949
apt-get update; \
5050
apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
5151
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
52-
52+
#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
53+
# COPY juicefs/juicefs /usr/local/bin/juicefs
54+
RUN wget --no-check-certificate "https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz" && tar -zxf "juicefs-1.0.2-linux-amd64.tar.gz" ;\
55+
mv juicefs /usr/local/bin/juicefs; \
56+
chmod +x /usr/local/bin/juicefs ; mkdir -p /run/sshd; \
57+
mkdir -p /root/.ssh; \
58+
chmod 700 /root/.ssh
5359
WORKDIR /
5460
COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
55-
ENTRYPOINT ["sidecar"]
61+
COPY script/*.sh /
62+
CMD [ "sidecar" ]

‎api/v1alpha1/backup_types.go

+11
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ import (
2121
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2222
)
2323

24+
type JuiceOpt struct {
25+
// sqlite or redis
26+
JuiceMeta string `json:"juiceMeta"`
27+
// backupSecrete name for S3
28+
BackupSecretName string `json:"backupSecretName"`
29+
JuiceName string `json:"juiceName"`
30+
}
31+
2432
// This is the backup Job CRD.
2533
// BackupSpec defines the desired state of Backup
2634
type BackupSpec struct {
@@ -40,6 +48,9 @@ type BackupSpec struct {
4048
// +optional
4149
NFSServerAddress string `json:"nfsServerAddress,omitempty"`
4250

51+
// Represents the juicefs parameters which need.
52+
// +optional
53+
JuiceOpt *JuiceOpt `json:"juiceOpt,omitempty"`
4354
// ClusterName represents the cluster name to backup
4455
ClusterName string `json:"clusterName"`
4556

‎api/v1alpha1/zz_generated.deepcopy.go

+20
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backup/syncer/job.go

+90
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,16 @@ limitations under the License.
1717
package syncer
1818

1919
import (
20+
"context"
2021
"fmt"
22+
"strings"
2123

2224
"github.com/presslabs/controller-util/pkg/syncer"
2325
batchv1 "k8s.io/api/batch/v1"
2426
corev1 "k8s.io/api/core/v1"
2527
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2628
"k8s.io/apimachinery/pkg/runtime"
29+
"k8s.io/apimachinery/pkg/types"
2730
"sigs.k8s.io/controller-runtime/pkg/client"
2831

2932
v1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
@@ -33,6 +36,7 @@ import (
3336
)
3437

3538
type jobSyncer struct {
39+
client client.Client
3640
job *batchv1.Job
3741
backup *backup.Backup
3842
}
@@ -50,6 +54,7 @@ func NewJobSyncer(c client.Client, backup *backup.Backup) syncer.Interface {
5054
}
5155

5256
sync := &jobSyncer{
57+
client: c,
5358
job: obj,
5459
backup: backup,
5560
}
@@ -174,6 +179,10 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
174179
MountPath: utils.XtrabckupLocal,
175180
},
176181
}
182+
} else if s.backup.Spec.JuiceOpt != nil {
183+
// Deal it for juiceOpt
184+
s.buildJuicefsBackPod(&in)
185+
177186
} else {
178187
// in.Containers[0].ImagePullPolicy = s.opt.ImagePullPolicy
179188
in.Containers[0].Args = []string{
@@ -238,3 +247,84 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
238247
}
239248
return in
240249
}
250+
251+
func (s *jobSyncer) buildJuicefsBackPod(in *corev1.PodSpec) error {
252+
// add volumn about pvc
253+
var defMode int32 = 0600
254+
var err error
255+
var cmdstr string
256+
in.Volumes = []corev1.Volume{
257+
{
258+
Name: utils.SShVolumnName,
259+
VolumeSource: corev1.VolumeSource{
260+
Secret: &corev1.SecretVolumeSource{
261+
SecretName: fmt.Sprintf("%s-ssh-key", s.backup.Spec.ClusterName),
262+
DefaultMode: &defMode,
263+
},
264+
},
265+
},
266+
}
267+
268+
in.Containers[0].VolumeMounts = []corev1.VolumeMount{
269+
{
270+
Name: utils.SShVolumnName,
271+
MountPath: utils.SshVolumnPath,
272+
},
273+
}
274+
275+
// PodName.clusterName-mysql.Namespace
276+
// sample-mysql-0.sample-mysql.default
277+
hostname := fmt.Sprintf("%s.%s-mysql.%s", s.backup.Spec.HostName, s.backup.Spec.ClusterName, s.backup.Namespace)
278+
if cmdstr, err = s.buildJuicefsCmd(s.backup.Spec.JuiceOpt.BackupSecretName); err != nil {
279+
return err
280+
}
281+
282+
in.Containers[0].Command = []string{"bash", "-c", "--", `cp /etc/secret-ssh/* /root/.ssh
283+
chmod 600 /root/.ssh/authorized_keys ;` +
284+
strings.Join([]string{
285+
"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", hostname, cmdstr,
286+
}, " ")}
287+
288+
return nil
289+
}
290+
291+
func (s *jobSyncer) buildJuicefsCmd(secName string) (string, error) {
292+
juiceopt := s.backup.Spec.JuiceOpt
293+
secret := &corev1.Secret{
294+
TypeMeta: metav1.TypeMeta{
295+
APIVersion: "v1",
296+
Kind: "Secret",
297+
},
298+
ObjectMeta: metav1.ObjectMeta{
299+
Name: secName,
300+
Namespace: s.backup.Namespace,
301+
},
302+
}
303+
err := s.client.Get(context.TODO(),
304+
types.NamespacedName{Namespace: s.backup.Namespace,
305+
Name: secName}, secret)
306+
307+
if err != nil {
308+
return "", err
309+
}
310+
url, bucket := secret.Data["s3-endpoint"], secret.Data["s3-bucket"]
311+
accesskey, secretkey := secret.Data["s3-access-key"], secret.Data["s3-secret-key"]
312+
juicebucket := utils.InstallBucket(string(url), string(bucket))
313+
cmdstr := fmt.Sprintf(`<<EOF
314+
export CLUSTER_NAME=%s
315+
juicefs format --storage s3 \
316+
--bucket %s \
317+
--access-key %s \
318+
--secret-key %s \
319+
%s \
320+
%s`, s.backup.Spec.ClusterName, juicebucket, accesskey, secretkey, juiceopt.JuiceMeta, juiceopt.JuiceName)
321+
cmdstr += fmt.Sprintf(`
322+
juicefs mount -d %s /%s/
323+
`, juiceopt.JuiceMeta, juiceopt.JuiceName)
324+
cmdstr += fmt.Sprintf(`
325+
source /backup.sh
326+
backup
327+
juicefs umount /%s/
328+
EOF`, juiceopt.JuiceName)
329+
return cmdstr, nil
330+
}

‎charts/mysql-operator/crds/mysql.radondb.com_backups.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,22 @@ spec:
7070
default: radondb/mysql57-sidecar:v2.3.0
7171
description: To specify the image that will be used for sidecar container.
7272
type: string
73+
juiceOpt:
74+
description: Represents the juicefs parameters which need.
75+
properties:
76+
backupSecretName:
77+
description: backupSecrete name for S3
78+
type: string
79+
juiceMeta:
80+
description: sqlite or redis
81+
type: string
82+
juiceName:
83+
type: string
84+
required:
85+
- backupSecretName
86+
- juiceMeta
87+
- juiceName
88+
type: object
7389
nfsServerAddress:
7490
description: Represents the ip address of the nfs server.
7591
type: string

‎config/crd/bases/mysql.radondb.com_backups.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,22 @@ spec:
7070
default: radondb/mysql57-sidecar:v2.3.0
7171
description: To specify the image that will be used for sidecar container.
7272
type: string
73+
juiceOpt:
74+
description: Represents the juicefs parameters which need.
75+
properties:
76+
backupSecretName:
77+
description: backupSecrete name for S3
78+
type: string
79+
juiceMeta:
80+
description: sqlite or redis
81+
type: string
82+
juiceName:
83+
type: string
84+
required:
85+
- backupSecretName
86+
- juiceMeta
87+
- juiceName
88+
type: object
7389
nfsServerAddress:
7490
description: Represents the ip address of the nfs server.
7591
type: string

‎controllers/mysqlcluster_controller.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ import (
2424
"github.com/presslabs/controller-util/pkg/syncer"
2525
appsv1 "k8s.io/api/apps/v1"
2626
corev1 "k8s.io/api/core/v1"
27-
policyv1beta1 "k8s.io/api/policy/v1beta1"
27+
28+
// policyv1beta1 "k8s.io/api/policy/v1beta1"
29+
policyv1beta1 "k8s.io/api/policy/v1"
2830
rbacv1 "k8s.io/api/rbac/v1"
2931
"k8s.io/apimachinery/pkg/api/errors"
3032
"k8s.io/apimachinery/pkg/labels"
@@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
125127
return ctrl.Result{}, err
126128
}
127129

130+
secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
131+
if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
132+
return ctrl.Result{}, err
133+
}
134+
128135
// Todo: modify mysql cm will trigger rolling update but it will not be applied.
129136
cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
130137
sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion

‎docs/en-us/juicefs_backup_and_restore.md

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
English | [简体中文](../zh-cn/juicefs_backup_and_restore.md) |
2+
3+
目录
4+
=============
5+
* [juiceopt备份](#开启juiceopt-备份)
6+
* [准备工作](#准备工作)
7+
* [填写配置](#填写backup-crd-的-yaml-配置信息)
8+
* [运行备份](#运行备份)
9+
10+
* [恢复](#恢复)
11+
* [准备工作](#恢复集群的准备工作)
12+
13+
# startup juiceopt backup
14+
## prerequire
15+
1. prepare S3 strorage （if you want other types of storage, reference the juicfs document），Obtain the access-key and secret-key . In the example of this article, it use minio ,and the instance is minio-1668754867, and the bucket named `test` so the url is http://test.minio-1668754867.minio:9000/ , you can modify it according your situations, and how to do you can refer to juicefs documents.
16+
17+
2. Install the redis , although juicefs also support sqlite as meta data storage, buf if you do so, you should sync the meta file from s3 at first,and I do not recommend it. redis url is the form as follow：
18+
```
19+
redis://<redis-server-name/IP>:<port>/<NO of database>
20+
```
21+
in the example of this article, redis-server-name is redis-leader, the number of database is 1, So the redis url is `redis://redis-leader:6379/1`
22+
23+
3. Verfiy whether it works: suppose the backup directory is juicefs , you can login in Pod's backup container , execute commanas as follow：
24+
25+
```
26+
juicefs format --storage s3 \
27+
--bucket http://test.minio-1668754867.minio:9000/ \
28+
--access-key <your access key> \
29+
--secret-key <your secrete key> > \
30+
redis://redis-leader:6379/1 \
31+
juicefs
32+
```
33+
then execute ：
34+
`juicefs mount -d redis://redis-leader:6379/1 /juicefs`
35+
36+
check whether juicefs is exist, write files, and check S3 storage whether has changed.
37+
38+
## fill backup crd's yaml configs
39+
In backup crd's yaml file, such as in samples/mysql_v1alpha_backup.yaml, add fields information under spec:
40+
41+
```
42+
juiceOpt:
43+
juiceMeta: <fill your redis url>
44+
backupSecretName: <S3's secret name, for example:sample-backup-secret>
45+
juiceName: <juicefs backup directory>
46+
```
47+
for example:
48+
```
49+
juiceOpt:
50+
juiceMeta: "redis://redis-leader:6379/1"
51+
backupSecretName: sample-backup-secret
52+
juiceName: juicefs
53+
```
54+
Others refer to [backup and restore config](./backup_and_restoration_s3.md)
55+
56+
## Running backup.
57+
58+
use command `kubectl apply -f <your backup crd's yaml>` , for examples：
59+
60+
```
61+
kubectl apply -f config/samples/mysql_v1alpha1_backup.yaml
62+
```
63+
64+
# Restore
65+
## prerequest for restore
66+
I suppose that the cluster you want restore is `sample2`
67+
### and `config map`
68+
1. At first give the `config map` a name,name's form is <name of restore cluster>-restore, this article suppose that cluster name is sample2, so `config map`'s name is `sample2-restore`
69+
2. Create config map
70+
* prepare for juiceopt parameters:
71+
build a yaml file, named `juiceopt.yaml`, fill it with:
72+
```
73+
juiceMeta: <redis url>
74+
backupSecretName: <S3's secret name>
75+
juiceName: <backup directory under S3 bucket>
76+
```
77+
for example, in the example of this article, juiceopt.yaml is:
78+
```
79+
juiceMeta: "redis://redis-leader:6379/1"
80+
backupSecretName: sample-backup-secret
81+
juiceName: juicefs
82+
```
83+
* use `kubectl create configmap` create a configmap
84+
configmap has two keys , `from` and `juice.opt` that respectively indicate the cluster has been backuped which we should restore from, and the juice parameter.
85+
but `date` key is optional, it indicates the time where restore to (format is:"2006-01-02 09:07:41"), if it does not have got this key , it will restore to now, use the commands as follows:
86+
`kubectl create configmap sample2-restore --from-literal=from=sample --from-file="juice.opt"=./juiceopt.yaml `
87+
88+
89+
### config mysql cluster's yaml
90+
in the example of this article, we suppose the cluster need to restore is sample2, the config method can refer to [radondb cluster configuration](./deploy_radondb-mysql_operator_on_k8s.md)
91+
### use kubectl apply the yaml
92+
use `kubectl apply ` apply the yaml file, for the example, use the commands as follow:
93+
94+
`kubectl apply -f config/samples/mysql_v1alpha1_mysqlcluster.yaml `
95+

0 commit comments

Comments
 (0)