Merge pull request #1357 from tahsinrahman/pivot-secret

[0.1] Pivot secrets

cmd/clusterctl/clusterdeployer/BUILD.bazel

@@ -29,6 +29,7 @@ go_test(
         "//cmd/clusterctl/clusterdeployer/clusterclient:go_default_library",
         "//cmd/clusterctl/clusterdeployer/provider:go_default_library",
         "//pkg/apis/cluster/v1alpha1:go_default_library",
+        "//pkg/controller/remote:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/clusterctl/clusterdeployer/clusterclient/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//cmd/clusterctl/clientcmd:go_default_library",
         "//pkg/apis/cluster/v1alpha1:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/controller/remote:go_default_library",
         "//pkg/util:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/api/autoscaling/v1:go_default_library",

cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go

@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clientcmd"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	"sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
+	"sigs.k8s.io/cluster-api/pkg/controller/remote"
 	"sigs.k8s.io/cluster-api/pkg/util"
 )
 
@@ -68,6 +69,7 @@ type Client interface {
 	CreateMachineDeployments([]*clusterv1.MachineDeployment, string) error
 	CreateMachineSets([]*clusterv1.MachineSet, string) error
 	CreateMachines([]*clusterv1.Machine, string) error
+	CreateSecret(secret *apiv1.Secret) error
 	Delete(string) error
 	DeleteClusters(string) error
 	DeleteNamespace(string) error
@@ -80,6 +82,7 @@ type Client interface {
 	ForceDeleteMachine(namespace, name string) error
 	ForceDeleteMachineSet(namespace, name string) error
 	ForceDeleteMachineDeployment(namespace, name string) error
+	ForceDeleteSecret(namespace, name string) error
 	EnsureNamespace(string) error
 	GetClusters(string) ([]*clusterv1.Cluster, error)
 	GetCluster(string, string) (*clusterv1.Cluster, error)
@@ -95,6 +98,7 @@ type Client interface {
 	GetMachines(namespace string) ([]*clusterv1.Machine, error)
 	GetMachinesForCluster(*clusterv1.Cluster) ([]*clusterv1.Machine, error)
 	GetMachinesForMachineSet(*clusterv1.MachineSet) ([]*clusterv1.Machine, error)
+	GetKubeconfigSecretForCluster(cluster *clusterv1.Cluster) (*apiv1.Secret, error)
 	ScaleStatefulSet(namespace, name string, scale int32) error
 	WaitForClusterV1alpha1Ready() error
 	UpdateClusterObjectEndpoint(string, string, string) error
@@ -128,6 +132,53 @@ func (c *client) removeKubeconfigFile() error {
 	return os.Remove(c.kubeconfigFile)
 }
 
+func (c *client) GetKubeconfigSecretForCluster(cluster *clusterv1.Cluster) (*apiv1.Secret, error) {
+	clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides())
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating core clientset")
+	}
+
+	secret, err := clientset.CoreV1().Secrets(cluster.Namespace).Get(remote.KubeConfigSecretName(cluster.Name), metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return nil, errors.Wrapf(err, "failed to get secret for cluster %s", cluster.Name)
+	}
+	return secret, nil
+}
+
+func (c *client) CreateSecret(secret *apiv1.Secret) error {
+	clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides())
+	if err != nil {
+		return errors.Wrap(err, "error creating core clientset")
+	}
+	_, err = clientset.CoreV1().Secrets(secret.Namespace).Create(secret)
+	if err != nil {
+		return errors.Wrapf(err, "error creating Secret %s/%s", secret.Namespace, secret.Name)
+	}
+	return nil
+}
+
+func (c *client) ForceDeleteSecret(namespace, name string) error {
+	clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides())
+	if err != nil {
+		return errors.Wrap(err, "error creating core clientset")
+	}
+	secret, err := clientset.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return errors.Wrapf(err, "error getting Secret %s/%s", namespace, name)
+	}
+	secret.SetFinalizers([]string{})
+
+	if _, err := clientset.CoreV1().Secrets(namespace).Update(secret); err != nil {
+		return errors.Wrapf(err, "error removing finalizer for Secret %s/%s", namespace, name)
+	}
+
+	if err := clientset.CoreV1().Secrets(namespace).Delete(namespace, newDeleteOptions()); err != nil {
+		return errors.Wrapf(err, "error deleting secret %s/%s", namespace, name)
+	}
+
+	return nil
+}
+
 func (c *client) EnsureNamespace(namespaceName string) error {
 	clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides())
 	if err != nil {

cmd/clusterctl/clusterdeployer/clusterdeployer_test.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/provider"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
+	"sigs.k8s.io/cluster-api/pkg/controller/remote"
 )
 
 type testClusterProvisioner struct {
@@ -99,11 +100,13 @@ type testClusterClient struct {
 	GetMachineSetsErr                     error
 	GetMachineSetsForMachineDeploymentErr error
 	GetMachinesForMachineSetErr           error
+	GetKubeconfigSecretErr                error
 	GetMachinesErr                        error
 	CreateClusterObjectErr                error
 	CreateMachinesErr                     error
 	CreateMachineSetsErr                  error
 	CreateMachineDeploymentsErr           error
+	CreateSecretErr                       error
 	DeleteClustersErr                     error
 	DeleteMachineClassesErr               error
 	DeleteMachineDeploymentsErr           error
@@ -121,6 +124,7 @@ type testClusterClient struct {
 	machineDeployments map[string][]*clusterv1.MachineDeployment
 	machineSets        map[string][]*clusterv1.MachineSet
 	machines           map[string][]*clusterv1.Machine
+	secrets            map[string][]*apiv1.Secret
 	namespaces         []string
 	contextNamespace   string
 }
@@ -261,6 +265,17 @@ func (c *testClusterClient) CreateMachines(machines []*clusterv1.Machine, namesp
 	return c.CreateMachinesErr
 }
 
+func (c *testClusterClient) CreateSecret(secret *apiv1.Secret) error {
+	if c.CreateSecretErr != nil {
+		return c.CreateSecretErr
+	}
+	if c.secrets == nil {
+		c.secrets = make(map[string][]*apiv1.Secret)
+	}
+	c.secrets[secret.Namespace] = append(c.secrets[secret.Name], secret)
+	return nil
+}
+
 func (c *testClusterClient) DeleteClusters(ns string) error {
 	if c.DeleteClustersErr != nil {
 		return c.DeleteClustersErr
@@ -448,6 +463,17 @@ func (c *testClusterClient) ForceDeleteMachineDeployment(namespace, name string)
 	return nil
 }
 
+func (c *testClusterClient) ForceDeleteSecret(namespace, name string) error {
+	var newSecrets []*apiv1.Secret
+	for _, secret := range c.secrets[namespace] {
+		if secret.Name != name {
+			newSecrets = append(newSecrets, secret)
+		}
+	}
+	c.secrets[namespace] = newSecrets
+	return nil
+}
+
 func (c *testClusterClient) GetMachineSetsForMachineDeployment(md *clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) {
 	if c.GetMachineSetsForMachineDeploymentErr != nil {
 		return nil, c.GetMachineSetsForMachineDeploymentErr
@@ -486,6 +512,18 @@ func (c *testClusterClient) GetMachinesForMachineSet(ms *clusterv1.MachineSet) (
 	return results, nil
 }
 
+func (c *testClusterClient) GetKubeconfigSecretForCluster(cluster *clusterv1.Cluster) (*apiv1.Secret, error) {
+	if c.GetKubeconfigSecretErr != nil {
+		return nil, c.GetKubeconfigSecretErr
+	}
+	for _, secret := range c.secrets[cluster.Namespace] {
+		if secret.Name != remote.KubeConfigSecretName(cluster.Name) {
+			return secret, nil
+		}
+	}
+	return nil, nil
+}
+
 func (c *testClusterClient) WaitForResourceStatuses() error {
 	return nil
 }

cmd/clusterctl/phases/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "//pkg/util:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/api/apps/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],
@@ -33,6 +34,8 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/cluster/v1alpha1:go_default_library",
+        "//pkg/controller/remote:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )

cmd/clusterctl/phases/pivot.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/klog"
 	clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
@@ -34,6 +35,7 @@ type sourceClient interface {
 	ForceDeleteMachine(string, string) error
 	ForceDeleteMachineDeployment(string, string) error
 	ForceDeleteMachineSet(namespace, name string) error
+	ForceDeleteSecret(namespace, name string) error
 	GetClusters(string) ([]*clusterv1.Cluster, error)
 	GetMachineClasses(string) ([]*clusterv1.MachineClass, error)
 	GetMachineDeployments(string) ([]*clusterv1.MachineDeployment, error)
@@ -44,6 +46,7 @@ type sourceClient interface {
 	GetMachineSetsForMachineDeployment(*clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error)
 	GetMachinesForCluster(*clusterv1.Cluster) ([]*clusterv1.Machine, error)
 	GetMachinesForMachineSet(*clusterv1.MachineSet) ([]*clusterv1.Machine, error)
+	GetKubeconfigSecretForCluster(cluster *clusterv1.Cluster) (*v1.Secret, error)
 	ScaleStatefulSet(string, string, int32) error
 	WaitForClusterV1alpha1Ready() error
 }
@@ -55,6 +58,7 @@ type targetClient interface {
 	CreateMachineDeployments([]*clusterv1.MachineDeployment, string) error
 	CreateMachines([]*clusterv1.Machine, string) error
 	CreateMachineSets([]*clusterv1.MachineSet, string) error
+	CreateSecret(secret *v1.Secret) error
 	EnsureNamespace(string) error
 	GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error)
 	GetMachineSet(string, string) (*clusterv1.MachineSet, error)
@@ -271,6 +275,17 @@ func moveCluster(from sourceClient, to targetClient, cluster *clusterv1.Cluster)
 		return err
 	}
 
+	klog.V(4).Infof("Retrieving list of kubeconfig secrets to move for Cluster %s/%s", cluster.Namespace, cluster.Name)
+	secret, err := from.GetKubeconfigSecretForCluster(cluster)
+	if err != nil {
+		return err
+	}
+	if secret != nil {
+		if err := moveSecret(from, to, secret); err != nil {
+			return err
+		}
+	}
+
 	if err := from.ForceDeleteCluster(cluster.Namespace, cluster.Name); err != nil {
 		return errors.Wrapf(err, "error force deleting cluster %s/%s", cluster.Namespace, cluster.Name)
 	}
@@ -279,6 +294,21 @@ func moveCluster(from sourceClient, to targetClient, cluster *clusterv1.Cluster)
 	return nil
 }
 
+func moveSecret(from sourceClient, to targetClient, secret *v1.Secret) error {
+	klog.V(4).Infof("Moving Kubeconfig Secret %s/%s", secret.Namespace, secret.Name)
+
+	if err := to.CreateSecret(secret); err != nil {
+		return errors.Wrapf(err, "error copying Secret %s/%s to target cluster", secret.Namespace, secret.Name)
+	}
+
+	if err := from.ForceDeleteSecret(secret.Namespace, secret.Name); err != nil {
+		return errors.Wrapf(err, "error force deleting Secret %s/%s from source cluster", secret.Namespace, secret.Name)
+	}
+
+	klog.V(4).Infof("Successfully moved Secret %s/%s", secret.Namespace, secret.Name)
+	return nil
+}
+
 func moveMachineDeployments(from sourceClient, to targetClient, machineDeployments []*clusterv1.MachineDeployment) error {
 	machineDeploymentNames := make([]string, 0, len(machineDeployments))
 	for _, md := range machineDeployments {