Skip to content

Commit da8dac9

Browse files
lancelot1989apecloud-bot
lancelot1989
authored and
apecloud-bot
committed
fix: cluster restore does not respect InstanceTemplate's ordinal ranges (#9552)
(cherry picked from commit 4d14454)
1 parent a925787 commit da8dac9

File tree

3 files changed

+233
-4
lines changed

3 files changed

+233
-4
lines changed

‎pkg/controller/plan/restore.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ func (r *RestoreManager) DoPrepareData(comp *component.SynthesizedComponent,
133133
for _, v := range comp.Instances {
134134
r.replicas = replicas(v)
135135
templateReplicas += r.replicas
136-
restore, err := r.BuildPrepareDataRestore(comp, backupObj, v.Name)
136+
restore, err := r.BuildPrepareDataRestore(comp, backupObj, &v)
137137
if err != nil {
138138
return err
139139
}
@@ -144,7 +144,7 @@ func (r *RestoreManager) DoPrepareData(comp *component.SynthesizedComponent,
144144
compReplicas := comp.Replicas - templateReplicas
145145
if compReplicas > 0 {
146146
r.replicas = compReplicas
147-
restore, err := r.BuildPrepareDataRestore(comp, backupObj, "")
147+
restore, err := r.BuildPrepareDataRestore(comp, backupObj, nil)
148148
if err != nil {
149149
return err
150150
}
@@ -155,7 +155,17 @@ func (r *RestoreManager) DoPrepareData(comp *component.SynthesizedComponent,
155155
return r.createRestoreAndWait(compObj, restores...)
156156
}
157157

158-
func (r *RestoreManager) BuildPrepareDataRestore(comp *component.SynthesizedComponent, backupObj *dpv1alpha1.Backup, templateName string) (*dpv1alpha1.Restore, error) {
158+
func (r *RestoreManager) BuildPrepareDataRestore(comp *component.SynthesizedComponent, backupObj *dpv1alpha1.Backup, template *appsv1.InstanceTemplate) (*dpv1alpha1.Restore, error) {
159+
templateName := ""
160+
startingIndex := r.startingIndex
161+
if template != nil {
162+
templateName = template.Name
163+
if len(template.Ordinals.Ranges) > 0 {
164+
// todo: currently restore api does not support multiple ranges, if implement in current way it
165+
// need to use multiple restore objects
166+
startingIndex = template.Ordinals.Ranges[0].Start
167+
}
168+
}
159169
backupMethod := backupObj.Status.BackupMethod
160170
if backupMethod == nil {
161171
return nil, intctrlutil.NewErrorf(intctrlutil.ErrorTypeRestoreFailed, `status.backupMethod of backup "%s" can not be empty`, backupObj.Name)
@@ -216,7 +226,7 @@ func (r *RestoreManager) BuildPrepareDataRestore(comp *component.SynthesizedComp
216226
VolumeClaimRestorePolicy: r.volumeRestorePolicy,
217227
RestoreVolumeClaimsTemplate: &dpv1alpha1.RestoreVolumeClaimsTemplate{
218228
Replicas: r.replicas,
219-
StartingIndex: r.startingIndex,
229+
StartingIndex: startingIndex,
220230
Templates: templates,
221231
},
222232
},

‎pkg/controller/plan/restore_test.go‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,207 @@ var _ = Describe("Restore", func() {
249249
restore.Status.Phase = dpv1alpha1.RestorePhaseCompleted
250250
})()).ShouldNot(HaveOccurred())
251251

252+
By("clean up annotations after cluster running")
253+
_ = restoreMGR.DoRestore(synthesizedComponent, compObj, true)
254+
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(cluster), func(g Gomega, tmpCluster *appsv1.Cluster) {
255+
g.Expect(tmpCluster.Annotations[constant.RestoreFromBackupAnnotationKey]).Should(BeEmpty())
256+
})).Should(Succeed())
257+
})
258+
})
259+
Context("Cluster InstanceTemplate ranges Restore", func() {
260+
const (
261+
compDefName = "test-compdef-1"
262+
defaultCompName = "mysql"
263+
topologyKey = "testTopologyKey"
264+
labelKey = "testNodeLabelKey"
265+
labelValue = "testLabelValue"
266+
)
267+
268+
var (
269+
compDef *appsv1.ComponentDefinition
270+
cluster *appsv1.Cluster
271+
synthesizedComponent *component.SynthesizedComponent
272+
compObj *appsv1.Component
273+
pvc *corev1.PersistentVolumeClaim
274+
backup *dpv1alpha1.Backup
275+
fullBackupActionSet *dpv1alpha1.ActionSet
276+
fullBackupActionSetName string
277+
startIndex = int32(10)
278+
)
279+
280+
BeforeEach(func() {
281+
By("By creating backup policyTemplate ")
282+
compDef = testapps.NewComponentDefinitionFactory(compDefName).
283+
SetDefaultSpec().
284+
Create(&testCtx).GetObject()
285+
286+
testdp.NewBackupPolicyTemplateFactory("backup-policy-template").
287+
SetCompDefs(compDef.Name).
288+
WithRandomName().
289+
AddBackupMethod(testdp.BackupMethodName, false, fullBackupActionSetName).
290+
SetBackupMethodVolumeMounts(testapps.DataVolumeName, "/data").Create(&testCtx).Get()
291+
292+
pvcSpec := testapps.NewPVCSpec("1Gi")
293+
cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, "").
294+
AddMultipleTemplateComponentRange(defaultCompName, compDefName).
295+
SetReplicas(3).
296+
AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec).
297+
Create(&testCtx).GetObject()
298+
299+
By("By mocking a pvc")
300+
pvc = testapps.NewPersistentVolumeClaimFactory(
301+
testCtx.DefaultNamespace, "data-"+clusterName+"-"+defaultCompName+"-0", clusterName, defaultCompName, "data").
302+
SetStorage("1Gi").
303+
Create(&testCtx).GetObject()
304+
305+
By("By mocking a pod")
306+
volume := corev1.Volume{Name: pvc.Name, VolumeSource: corev1.VolumeSource{
307+
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name}}}
308+
_ = testapps.NewPodFactory(testCtx.DefaultNamespace, clusterName+"-"+defaultCompName+"-0").
309+
AddAppInstanceLabel(clusterName).
310+
AddAppComponentLabel(defaultCompName).
311+
AddAppManagedByLabel().
312+
AddVolume(volume).
313+
AddContainer(corev1.Container{Name: testapps.DefaultMySQLContainerName, Image: testapps.ApeCloudMySQLImage}).
314+
AddNodeName("fake-node-name").
315+
Create(&testCtx).GetObject()
316+
317+
By("create actionset of full backup")
318+
fullBackupActionSet = testapps.CreateCustomizedObj(&testCtx, "backup/actionset.yaml", &dpv1alpha1.ActionSet{}, testapps.RandomizedObjName())
319+
fullBackupActionSetName = fullBackupActionSet.Name
320+
synthesizedComponent = &component.SynthesizedComponent{
321+
PodSpec: &compDef.Spec.Runtime,
322+
VolumeClaimTemplates: intctrlutil.ToCoreV1PVCTs(cluster.Spec.ComponentSpecs[0].VolumeClaimTemplates),
323+
Name: defaultCompName,
324+
Replicas: 3,
325+
Roles: []appsv1.ReplicaRole{
326+
{
327+
Name: "leader",
328+
UpdatePriority: 2,
329+
},
330+
{
331+
Name: "follower",
332+
UpdatePriority: 1,
333+
},
334+
},
335+
Instances: []appsv1.InstanceTemplate{{
336+
Name: "foo",
337+
Replicas: func() *int32 { replicas := int32(1); return &replicas }(),
338+
Ordinals: appsv1.Ordinals{
339+
Ranges: []appsv1.Range{
340+
{Start: startIndex, End: 20},
341+
},
342+
},
343+
}},
344+
}
345+
By("create component object")
346+
compObj = testapps.NewComponentFactory(testCtx.DefaultNamespace, cluster.Name+"-"+synthesizedComponent.Name, "").
347+
AddAnnotations(constant.KBAppClusterUIDKey, string(cluster.UID)).
348+
AddLabels(constant.AppInstanceLabelKey, cluster.Name).
349+
SetReplicas(1).
350+
Create(&testCtx).
351+
GetObject()
352+
353+
By("By creating remote pvc: ")
354+
remotePVC := testapps.NewPersistentVolumeClaimFactory(
355+
testCtx.DefaultNamespace, "remote-pvc", clusterName, defaultCompName, "log").
356+
SetStorage("1Gi").
357+
Create(&testCtx).GetObject()
358+
359+
By("By creating base backup: ")
360+
backupLabels := map[string]string{
361+
constant.AppInstanceLabelKey: sourceCluster,
362+
constant.KBAppComponentLabelKey: defaultCompName,
363+
}
364+
backup = testdp.NewBackupFactory(testCtx.DefaultNamespace, backupName).
365+
WithRandomName().SetLabels(backupLabels).
366+
SetBackupPolicyName("test-fake").
367+
SetBackupMethod(testdp.VSBackupMethodName).
368+
Create(&testCtx).GetObject()
369+
baseStartTime := &startTime
370+
baseStopTime := &now
371+
backup.Status = dpv1alpha1.BackupStatus{
372+
Phase: dpv1alpha1.BackupPhaseCompleted,
373+
StartTimestamp: baseStartTime,
374+
CompletionTimestamp: baseStopTime,
375+
PersistentVolumeClaimName: remotePVC.Name,
376+
}
377+
testdp.MockBackupStatusMethod(backup, testdp.VSBackupMethodName, testapps.DataVolumeName, testdp.ActionSetName)
378+
patchBackupStatus(backup.Status, client.ObjectKeyFromObject(backup))
379+
})
380+
381+
It("Test restore", func() {
382+
By("restore from backup")
383+
restoreFromBackup := fmt.Sprintf(`{"%s": {"name":"%s"}}`, defaultCompName, backup.Name)
384+
Expect(testapps.ChangeObj(&testCtx, cluster, func(tmpCluster *appsv1.Cluster) {
385+
tmpCluster.Annotations = map[string]string{
386+
constant.RestoreFromBackupAnnotationKey: restoreFromBackup,
387+
}
388+
})).Should(Succeed())
389+
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)).Should(Succeed())
390+
restoreMGR := NewRestoreManager(ctx, k8sClient, cluster, scheme.Scheme, nil, 3, 0)
391+
392+
By("restore from template")
393+
err := restoreMGR.DoRestore(synthesizedComponent, compObj, true)
394+
Expect(intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeNeedWaiting)).Should(BeTrue())
395+
396+
By("verify template restore")
397+
restore := &dpv1alpha1.Restore{}
398+
restoreMeta := restoreMGR.GetRestoreObjectMeta(synthesizedComponent, dpv1alpha1.PrepareData, "foo")
399+
namedspace := types.NamespacedName{Name: restoreMeta.Name, Namespace: restoreMeta.Namespace}
400+
Expect(k8sClient.Get(ctx, namedspace, restore)).Should(Succeed())
401+
Expect(restore.Spec.PrepareDataConfig.RestoreVolumeClaimsTemplate.StartingIndex).Should(Equal(startIndex))
402+
Expect(restore.Spec.PrepareDataConfig.RestoreVolumeClaimsTemplate.Replicas).Should(Equal(int32(1)))
403+
404+
By("mock template restore of prepareData stage to Completed")
405+
Expect(testapps.GetAndChangeObjStatus(&testCtx, namedspace, func(restore *dpv1alpha1.Restore) {
406+
restore.Status.Phase = dpv1alpha1.RestorePhaseCompleted
407+
})()).ShouldNot(HaveOccurred())
408+
409+
By("restore from default")
410+
err = restoreMGR.DoRestore(synthesizedComponent, compObj, true)
411+
Expect(intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeNeedWaiting)).Should(BeTrue())
412+
413+
By("verify default restore")
414+
defaultMeta := restoreMGR.GetRestoreObjectMeta(synthesizedComponent, dpv1alpha1.PrepareData, "")
415+
defaultNs := types.NamespacedName{Name: defaultMeta.Name, Namespace: defaultMeta.Namespace}
416+
defaultRestore := &dpv1alpha1.Restore{}
417+
Expect(k8sClient.Get(ctx, defaultNs, defaultRestore)).Should(Succeed())
418+
Expect(defaultRestore.Spec.PrepareDataConfig.RestoreVolumeClaimsTemplate.StartingIndex).Should(Equal(int32(0)))
419+
Expect(defaultRestore.Spec.PrepareDataConfig.RestoreVolumeClaimsTemplate.Replicas).Should(Equal(int32(2)))
420+
421+
By("mock default restore of prepareData stage to Completed")
422+
Expect(testapps.GetAndChangeObjStatus(&testCtx, defaultNs, func(restore *dpv1alpha1.Restore) {
423+
restore.Status.Phase = dpv1alpha1.RestorePhaseCompleted
424+
})()).ShouldNot(HaveOccurred())
425+
426+
By("mock component and cluster phase to Running")
427+
Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() {
428+
cluster.Status.Phase = appsv1.RunningClusterPhase
429+
cluster.Status.Components = map[string]appsv1.ClusterComponentStatus{
430+
defaultCompName: {
431+
Phase: appsv1.RunningComponentPhase,
432+
},
433+
}
434+
})).Should(Succeed())
435+
Expect(testapps.ChangeObjStatus(&testCtx, compObj, func() {
436+
compObj.Status.Phase = appsv1.RunningComponentPhase
437+
})).Should(Succeed())
438+
439+
By("wait for postReady restore created and mock it to Completed")
440+
restoreMGR.Cluster = cluster
441+
_ = restoreMGR.DoRestore(synthesizedComponent, compObj, true)
442+
443+
// check if restore CR of postReady stage is created.
444+
restoreMeta = restoreMGR.GetRestoreObjectMeta(synthesizedComponent, dpv1alpha1.PostReady, "")
445+
namedspace = types.NamespacedName{Name: restoreMeta.Name, Namespace: restoreMeta.Namespace}
446+
Eventually(testapps.CheckObjExists(&testCtx, namedspace,
447+
&dpv1alpha1.Restore{}, true)).Should(Succeed())
448+
// set restore to Completed
449+
Expect(testapps.GetAndChangeObjStatus(&testCtx, namedspace, func(restore *dpv1alpha1.Restore) {
450+
restore.Status.Phase = dpv1alpha1.RestorePhaseCompleted
451+
})()).ShouldNot(HaveOccurred())
452+
252453
By("clean up annotations after cluster running")
253454
_ = restoreMGR.DoRestore(synthesizedComponent, compObj, true)
254455
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(cluster), func(g Gomega, tmpCluster *appsv1.Cluster) {

‎pkg/testutil/apps/cluster_factory.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,24 @@ func (factory *MockClusterFactory) AddMultipleTemplateComponent(compName string,
9595
return factory
9696
}
9797

98+
func (factory *MockClusterFactory) AddMultipleTemplateComponentRange(compName string, compDefName string) *MockClusterFactory {
99+
comp := appsv1.ClusterComponentSpec{
100+
Name: compName,
101+
ComponentDef: compDefName,
102+
Instances: []appsv1.InstanceTemplate{{
103+
Name: "foo",
104+
Replicas: func() *int32 { replicas := int32(1); return &replicas }(),
105+
Ordinals: appsv1.Ordinals{
106+
Ranges: []appsv1.Range{
107+
{Start: 10, End: 20},
108+
},
109+
},
110+
}},
111+
}
112+
factory.Get().Spec.ComponentSpecs = append(factory.Get().Spec.ComponentSpecs, comp)
113+
return factory
114+
}
115+
98116
func (factory *MockClusterFactory) AddInstances(compName string, instance appsv1.InstanceTemplate) *MockClusterFactory {
99117
for i, compSpec := range factory.Get().Spec.ComponentSpecs {
100118
if compSpec.Name != compName {

0 commit comments

Comments
 (0)