Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added helper function to take multiple backups efficiently #2736

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Added helper function to take multiple backups efficiently #2736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5e0648
Added helper function to create multiple backups per deployment
sabrarhussaini Aug 7, 2024
81cc652
Modified the approach
sabrarhussaini Aug 8, 2024
fdb374c
Refactored the code
sabrarhussaini Aug 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 117 additions & 0 deletions tests/backup/backup_scale_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,3 +428,120 @@ var _ = Describe("{ValidateFiftyVolumeBackups}", Label(TestCaseLabelsMap[Validat
log.FailOnError(err, "Unable to switch context to source cluster [%s]", SourceClusterName)
})
})

var _ = Describe("{CreateMultipleBackupsPerDeployment}", func() {
var (
appNamespaces []string
backupAppContexts []*scheduler.Context
cloudAccountName string
cloudAccountUid string
backupLocationName string
backupLocationUid string
backupLocationMap map[string]string
backupNameList []string
srcClusterUid string
destClusterUid string
numDeployments = 25
numOfBackupsPerDeployment = 10
snapshotLimit = 3
)
JustBeforeEach(func() {
StartPxBackupTorpedoTest("CreateMultipleBackupsPerDeployment", "TC to verify multiple backups", nil, 0, Sabrarhussaini, Q2FY25)
log.Infof("Scheduling applications")
})

It("TC to verify multiple backups", func() {
Step("Provision apps for validation", func() {
log.InfoD("Provisioning apps for validation")
appList := Inst().AppList
defer func() {
Inst().AppList = appList
}()
Inst().AppList = []string{"postgres-backup"}
backupAppContexts = make([]*scheduler.Context, 0)
appNamespaces = make([]string, 0)
err := SetSourceKubeConfig()
log.FailOnError(err, "Switching context to source cluster failed")
log.Infof("Scheduling applications")
for i := 0; i < numDeployments; i++ {
taskName := fmt.Sprintf("multiple-%d", i)
appContexts := ScheduleApplications(taskName)
for _, appCtx := range appContexts {
namespace := GetAppNamespace(appCtx, taskName)
appNamespaces = append(appNamespaces, namespace)
backupAppContexts = append(backupAppContexts, appCtx)
appCtx.ReadinessTimeout = AppReadinessTimeout
}
}
})

Step("Validating applications ", func() {
log.InfoD("Validating applications")
ValidateApplications(backupAppContexts)
})

Step("Create cloud credentials and backup locations", func() {
log.InfoD("Create cloud credentials and backup locations")
providers := GetBackupProviders()
backupLocationMap = make(map[string]string)
ctx, err := backup.GetAdminCtxFromSecret()
log.FailOnError(err, "Fetching px-central-admin ctx")
for _, provider := range providers {
cloudAccountUid = uuid.New()
cloudAccountName = fmt.Sprintf("%s-%s-%v", "cred-partial-bkp", provider, time.Now().Unix())
log.Infof("Creating a cloud credential for partial backup [%s] with UID [%s] using [%s] as the provider", cloudAccountUid, cloudAccountName, provider)
err := CreateCloudCredential(provider, cloudAccountName, cloudAccountUid, BackupOrgID, ctx)
dash.VerifyFatal(err, nil, fmt.Sprintf("Verifying creation of cloud credential [%s] with UID [%s] using [%s] as the provider", cloudAccountName, BackupOrgID, provider))
backupLocationName = fmt.Sprintf("%s-partial-bl-%v", getGlobalBucketName(provider), time.Now().Unix())
backupLocationUid = uuid.New()
backupLocationMap[backupLocationUid] = backupLocationName
bucketName := getGlobalBucketName(provider)
log.Infof("Creating a backup location [%s] with UID [%s] using the [%s] bucket", backupLocationName, backupLocationUid, bucketName)
err = CreateBackupLocation(provider, backupLocationName, backupLocationUid, cloudAccountName, cloudAccountUid, bucketName, BackupOrgID, "", true)
dash.VerifyFatal(err, nil, fmt.Sprintf("Verifying creation of backup location [%s] with UID [%s] using the bucket [%s]", backupLocationName, backupLocationUid, bucketName))
}
})

Step("Create source and destination clusters", func() {
log.InfoD("Creating source and destination clusters")
ctx, err := backup.GetAdminCtxFromSecret()
log.FailOnError(err, "Fetching px-central-admin ctx")
log.Infof("Creating source [%s] and destination [%s] clusters", SourceClusterName, DestinationClusterName)
err = CreateApplicationClusters(BackupOrgID, "", "", ctx)
dash.VerifyFatal(err, nil, fmt.Sprintf("Verifying creation of source [%s] and destination [%s] clusters with px-central-admin ctx", SourceClusterName, DestinationClusterName))
srcClusterStatus, err := Inst().Backup.GetClusterStatus(BackupOrgID, SourceClusterName, ctx)
log.FailOnError(err, fmt.Sprintf("Fetching [%s] cluster status", SourceClusterName))
dash.VerifyFatal(srcClusterStatus, api.ClusterInfo_StatusInfo_Online, fmt.Sprintf("Verifying if [%s] cluster is online", SourceClusterName))
srcClusterUid, err = Inst().Backup.GetClusterUID(ctx, BackupOrgID, SourceClusterName)
log.FailOnError(err, fmt.Sprintf("Fetching [%s] cluster uid", SourceClusterName))
log.Infof("Cluster [%s] uid: [%s]", SourceClusterName, srcClusterUid)
dstClusterStatus, err := Inst().Backup.GetClusterStatus(BackupOrgID, DestinationClusterName, ctx)
log.FailOnError(err, fmt.Sprintf("Fetching [%s] cluster status", DestinationClusterName))
dash.VerifyFatal(dstClusterStatus, api.ClusterInfo_StatusInfo_Online, fmt.Sprintf("Verifying if [%s] cluster is online", DestinationClusterName))
destClusterUid, err = Inst().Backup.GetClusterUID(ctx, BackupOrgID, DestinationClusterName)
log.FailOnError(err, fmt.Sprintf("Fetching [%s] cluster uid", DestinationClusterName))
log.Infof("Cluster [%s] uid: [%s]", DestinationClusterName, destClusterUid)
})

Step("Taking multiple backups of each namespace for the application cluster", func() {
log.InfoD("Taking backup of application for the clusters")
ctx, err := backup.GetAdminCtxFromSecret()
log.FailOnError(err, "Fetching px-central-admin ctx")
backupNameList, err = TakeMultipleBackupsPerDeployment(ctx, BackupOrgID, SourceClusterName, numOfBackupsPerDeployment, snapshotLimit, backupLocationName, backupLocationUid, backupAppContexts, BackupNamePrefix)
log.FailOnError(err, "Taking backup of cluster failed")
log.Infof("Backups taken: %v", backupNameList)
})
})

JustAfterEach(func() {
defer EndPxBackupTorpedoTest(backupAppContexts)
ctx, err := backup.GetAdminCtxFromSecret()
log.FailOnError(err, "Fetching px-central-admin ctx")
err = SetSourceKubeConfig()
log.FailOnError(err, "Switching context to source cluster failed")
opts := make(map[string]bool)
opts[SkipClusterScopedObjects] = true
DestroyApps(backupAppContexts, opts)
CleanupCloudSettingsAndClusters(backupLocationMap, cloudAccountName, cloudAccountUid, ctx)
})
})
102 changes: 102 additions & 0 deletions tests/backup_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1006,6 +1006,108 @@ func CreateBackupWithCustomResourceTypeWithValidation(ctx context1.Context, back
return ValidateBackup(ctx, backupName, orgID, scheduledAppContextsToBackup, resourceTypesFilter)
}

// TakeMultipleBackupsPerDeployment takes multiple backups for each deployment in the specified cluster.
func TakeMultipleBackupsPerDeployment(ctx context1.Context, backupOrgID, clusterName string, numOfBackups, snapShotLimit int, backupLocationName, backupLocationUid string, scheduledAppContextsToBackup []*scheduler.Context, backupNamePrefix string) ([]string, error) {
labelSelectors := make(map[string]string)
type backupResult struct {
name string
err error
bkpContext *scheduler.Context
}
ctx, cancel := context1.WithCancel(ctx)
defer cancel()

var (
backupNameList []string
errList []error
wg sync.WaitGroup
backupResults = make(chan backupResult)
)
clusterUid, err := Inst().Backup.GetClusterUID(ctx, backupOrgID, clusterName)
if err != nil {
return backupNameList, err
}

for _, scheduledAppContext := range scheduledAppContextsToBackup {
wg.Add(1)
go func(scheduledAppContext *scheduler.Context) {
defer wg.Done()
defer GinkgoRecover()

var innerWg sync.WaitGroup
semaphore := make(chan struct{}, snapShotLimit)
for i := 0; i < numOfBackups; i++ {
innerWg.Add(1)
go func(i int) {
defer innerWg.Done()
defer GinkgoRecover()

select {
case semaphore <- struct{}{}:
defer func() { <-semaphore }()
case <-ctx.Done():
return
}
currentBackupName := fmt.Sprintf("%s-%s-%d", backupNamePrefix, RandomString(8), i+1)
err := CreateBackup(currentBackupName, clusterName, backupLocationName, backupLocationUid, []string{scheduledAppContext.ScheduleOptions.Namespace}, labelSelectors, backupOrgID, clusterUid, "", "", "", "", ctx)
backupResults <- backupResult{name: currentBackupName, err: err, bkpContext: scheduledAppContext}
}(i)
}

innerWg.Wait()
}(scheduledAppContext)
}
go func() {
wg.Wait()
close(backupResults)
}()

backupMap := make(map[string]*scheduler.Context)
for result := range backupResults {
if result.err != nil {
log.Errorf("Failed to create backup: %v", result.err)
errList = append(errList, result.err)
cancel()
break
} else {
log.Infof("Successfully created backup [%s]", result.name)
backupNameList = append(backupNameList, result.name)
backupMap[result.name] = result.bkpContext
}
}

if len(errList) > 0 {
return backupNameList, fmt.Errorf("some backups failed: %v", errList)
}
validationResults := make(chan error)

// Validate all the backups.
for backupName, bkpContext := range backupMap {
wg.Add(1)
go func(backupName string, bkpContext *scheduler.Context) {
defer wg.Done()
defer GinkgoRecover()

err := ValidateBackup(ctx, backupName, backupOrgID, []*scheduler.Context{bkpContext}, nil)
validationResults <- err
}(backupName, bkpContext)
}
go func() {
wg.Wait()
close(validationResults)
}()
for err := range validationResults {
if err != nil {
log.Errorf("Validation failed for backup: %v", err)
errList = append(errList, err)
}
}
if len(errList) > 0 {
return backupNameList, fmt.Errorf("some validations failed : %v", errList)
}
return backupNameList, nil
}

// CreateScheduleBackup creates a schedule backup and checks for success of first (immediately triggered) backup
func CreateScheduleBackup(scheduleName string, clusterName string, bLocation string, bLocationUID string,
namespaces []string, labelSelectors map[string]string, orgID string, preRuleName string,
Expand Down