Watch scheduler CR and make masters schedulable accordingly

By default, the MCO's kubelet configuration injects a taint to disable scheduling. This adds
support in the MCO for watching the API recently added to allow the masters
to be schedulable. openshift/api#366

Use cases here are for 3 node bare metal clusters (which have significant resources on the masters),
as well as CodeReady Containers which wants to make a single node OpenShift as a VM.

Closes: openshift#763

cmd/machine-config-controller/start.go

@@ -137,6 +137,7 @@ func createControllers(ctx *controllercommon.ControllerContext) []controllercomm
 		node.New(
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
 			ctx.KubeInformerFactory.Core().V1().Nodes(),
+			ctx.ConfigInformerFactory.Config().V1().Schedulers(),
 			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),
 		),

internal/clients/builder.go

@@ -41,8 +41,7 @@ func (cb *Builder) KubeClient(name string) (kubernetes.Interface, error) {
 	return kubernetes.NewForConfig(rest.AddUserAgent(cb.config, name))
 }
 
-// ConfigClientOrDie returns the kubernetes client interface for security related kubernetes objects
-// such as pod security policy, security context.
+// ConfigClientOrDie returns the config client interface for openshift
 func (cb *Builder) ConfigClientOrDie(name string) configclientset.Interface {
 	return configclientset.NewForConfigOrDie(rest.AddUserAgent(cb.config, name))
 }

internal/kubeutils.go

@@ -0,0 +1,52 @@
+package internal
+
+import (
+	"encoding/json"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1lister "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/util/retry"
+)
+
+// UpdateNodeRetry calls f to update a node object in Kubernetes.
+// It will attempt to update the node by applying f to it up to DefaultBackoff
+// number of times.
+// f will be called each time since the node object will likely have changed if
+// a retry is necessary.
+func UpdateNodeRetry(client corev1client.NodeInterface, lister corev1lister.NodeLister, nodeName string, f func(*corev1.Node)) (*corev1.Node, error) {
+	var node *corev1.Node
+	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		n, err := lister.Get(nodeName)
+		if err != nil {
+			return err
+		}
+		oldNode, err := json.Marshal(n)
+		if err != nil {
+			return err
+		}
+
+		nodeClone := n.DeepCopy()
+		f(nodeClone)
+
+		newNode, err := json.Marshal(nodeClone)
+		if err != nil {
+			return err
+		}
+
+		patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldNode, newNode, corev1.Node{})
+		if err != nil {
+			return fmt.Errorf("failed to create patch for node %q: %v", nodeName, err)
+		}
+
+		node, err = client.Patch(nodeName, types.StrategicMergePatchType, patchBytes)
+		return err
+	}); err != nil {
+		// may be conflict if max retries were hit
+		return nil, fmt.Errorf("unable to update node %q: %v", node, err)
+	}
+	return node, nil
+}

manifests/machineconfigcontroller/clusterrole.yaml

@@ -14,7 +14,7 @@ rules:
   resources: ["configmaps", "secrets"]
   verbs: ["*"]
 - apiGroups: ["config.openshift.io"]
-  resources: ["images", "clusterversions", "featuregates"]
+  resources: ["images", "clusterversions", "featuregates", "schedulers"]
   verbs: ["*"]
 - apiGroups: ["operator.openshift.io"]
   resources: ["imagecontentsourcepolicies"]

pkg/controller/kubelet-config/helpers.go

@@ -41,7 +41,9 @@ func createNewKubeletIgnition(jsonConfig []byte) igntypes.Config {
 func createNewDefaultFeatureGate() *osev1.FeatureGate {
 	return &osev1.FeatureGate{
 		Spec: osev1.FeatureGateSpec{
-			FeatureSet: osev1.Default,
+			FeatureGateSelection: osev1.FeatureGateSelection{
+				FeatureSet: osev1.Default,
+			},
 		},
 	}
 }

pkg/controller/kubelet-config/kubelet_config_controller_test.go

@@ -92,7 +92,7 @@ func newFeatures(name string, enabled, disabled []string, labels map[string]stri
 	return &osev1.FeatureGate{
 		TypeMeta:   metav1.TypeMeta{APIVersion: osev1.GroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels, UID: types.UID(utilrand.String(5))},
-		Spec:       osev1.FeatureGateSpec{FeatureSet: ""},
+		Spec:       osev1.FeatureGateSpec{FeatureGateSelection: osev1.FeatureGateSelection{FeatureSet: ""}},
 	}
 }
 

pkg/controller/kubelet-config/kubelet_config_features.go

@@ -62,7 +62,9 @@ func (ctrl *Controller) syncFeatureHandler(key string) error {
 		glog.V(2).Infof("FeatureSet %v is missing, using default", key)
 		features = &osev1.FeatureGate{
 			Spec: osev1.FeatureGateSpec{
-				FeatureSet: osev1.Default,
+				FeatureGateSelection: osev1.FeatureGateSelection{
+					FeatureSet: osev1.Default,
+				},
 			},
 		}
 	} else if err != nil {