[release-1.7] 🌱 Add retry to clusterctl UpgradeWithBinary (#11542)

* seedling: Add retry to clusterctl `UpgradeWithBinary`

* fix for go 1.21

---------

Co-authored-by: Stephen Cahill <[email protected]>
Co-authored-by: Christian Schlotter <[email protected]>

Author: 3 people

test/framework/clusterctl/client.go

@@ -203,7 +203,7 @@ func Upgrade(ctx context.Context, input UpgradeInput) {
 }
 
 // UpgradeWithBinary calls clusterctl upgrade apply with the list of providers defined in the local repository.
-func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) {
+func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) error {
 	if len(input.ClusterctlVariables) > 0 {
 		outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName))
 		Expect(CopyAndAmendClusterctlConfig(ctx, CopyAndAmendClusterctlConfigInput{
@@ -227,8 +227,9 @@ func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) {
 		if errors.As(err, &exitErr) {
 			stdErr = string(exitErr.Stderr)
 		}
+		return fmt.Errorf("failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
 	}
-	Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr)
+	return nil
 }
 
 func calculateClusterCtlUpgradeArgs(input UpgradeInput) []string {

test/framework/clusterctl/clusterctl_helpers.go

@@ -196,12 +196,27 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen
 	client := input.ClusterProxy.GetClient()
 
 	if input.ClusterctlBinaryPath != "" {
-		UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput)
+		clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath)
+		Expect(err).ToNot(HaveOccurred())
+		upgradeRetries := 1
+		// Older versions of clusterctl may need to retry the upgrade process to allow for
+		// cert-manager CAs to become available before continuing.  For newer versions of clusterctl
+		// this is addressed with https://github.com/kubernetes-sigs/cluster-api/pull/10513
+		if clusterctlVersion.LT(semver.MustParse("1.7.0")) {
+			upgradeRetries = 2
+		}
+		for i := 0; i < upgradeRetries; i++ {
+			err := UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput)
+			if err != nil && i < upgradeRetries-1 {
+				log.Logf("Failed to UpgradeWithBinary, retrying: %v", err)
+				continue
+			}
+			Expect(err).ToNot(HaveOccurred())
+			break
+		}
 		// Old versions of clusterctl may deploy CRDs, Mutating- and/or ValidatingWebhookConfigurations
 		// before creating the new Certificate objects. This check ensures the CA's are up to date before
 		// continuing.
-		clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath)
-		Expect(err).ToNot(HaveOccurred())
 		if clusterctlVersion.LT(semver.MustParse("1.7.2")) {
 			Eventually(func() error {
 				return verifyCAInjection(ctx, client)