MachineHealthCheck supports checking Machine conditions

MachineHealthCheck currently only allows checking Node conditions to
validate if a machine is healthy. However, machine conditions capture
conditions that do not exist on nodes, for example, control plane node
conditions such as EtcdPodHealthy, SchedulerPodHealthy that can indicate
if a controlplane machine has been created correctly.

Adding support for Machine conditions enables us to perform remediation
during control plane upgrades.

This PR introduces a new fieldas part of the MachineHealthCheckSpec:
  - `UnhealthyMachineConditions`

This will mirror the behavior of `UnhealthyNodeConditions` but the
MachineHealthCheck controller will instead check the machine conditions.

Author: justinmir

api/core/v1beta1/machinehealthcheck_types.go

@@ -69,6 +69,14 @@ type MachineHealthCheckSpec struct {
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyConditions []UnhealthyCondition `json:"unhealthyConditions,omitempty"`
 
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a node is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the node is unhealthy.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
+
 	// maxUnhealthy specifies the maximum number of unhealthy machines allowed.
 	// Any further remediation is only allowed if at most "maxUnhealthy" machines selected by
 	// "selector" are not healthy.
@@ -148,6 +156,34 @@ type UnhealthyCondition struct {
 
 // ANCHOR_END: UnhealthyCondition
 
+// ANCHOR: UnhealthyMachineCondition
+
+// UnhealthyMachineCondition represents a Node condition type and value with a timeout
+// specified as a duration.  When the named condition has been in the given
+// status for at least the timeout value, a node is considered unhealthy.
+type UnhealthyMachineCondition struct {
+	// type of Node condition
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Type string `json:"type"`
+
+	// status of the condition, one of True, False, Unknown.
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Status metav1.ConditionStatus `json:"status"`
+
+	// timeout is the duration that a node must be in a given status for,
+	// after which the node is considered unhealthy.
+	// For example, with a value of "1h", the node must match the status
+	// for at least 1 hour before being considered unhealthy.
+	// +required
+	Timeout metav1.Duration `json:"timeout"`
+}
+
+// ANCHOR_END: UnhealthyMachineCondition
+
 // ANCHOR: MachineHealthCheckStatus
 
 // MachineHealthCheckStatus defines the observed state of MachineHealthCheck.

api/core/v1beta1/zz_generated.conversion.go

@@ -276,6 +276,10 @@ const (
 	// defined by a MachineHealthCheck object.
 	MachineHealthCheckUnhealthyNodeReason = "UnhealthyNode"
 
+	// MachineHealthCheckUnhealthyMachineReason surfaces when the machine does not pass the health checks
+	// defined by a MachineHealthCheck object.
+	MachineHealthCheckUnhealthyMachineReason = "UnhealthyMachine"
+
 	// MachineHealthCheckNodeStartupTimeoutReason surfaces when the node hosted on the machine does not appear within
 	// the timeout defined by a MachineHealthCheck object.
 	MachineHealthCheckNodeStartupTimeoutReason = "NodeStartupTimeout"

api/core/v1beta1/zz_generated.deepcopy.go

@@ -69,6 +69,14 @@ type MachineHealthCheckSpec struct {
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
 
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a node is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the node is unhealthy.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
+
 	// maxUnhealthy specifies the maximum number of unhealthy machines allowed.
 	// Any further remediation is only allowed if at most "maxUnhealthy" machines selected by
 	// "selector" are not healthy.
@@ -148,6 +156,34 @@ type UnhealthyNodeCondition struct {
 
 // ANCHOR_END: UnhealthyNodeCondition
 
+// ANCHOR: UnhealthyMachineCondition
+
+// UnhealthyMachineCondition represents a Node condition type and value with a timeout
+// specified as a duration.  When the named condition has been in the given
+// status for at least the timeout value, a node is considered unhealthy.
+type UnhealthyMachineCondition struct {
+	// type of Node condition
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Type string `json:"type"`
+
+	// status of the condition, one of True, False, Unknown.
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:MinLength=1
+	// +required
+	Status metav1.ConditionStatus `json:"status"`
+
+	// timeout is the duration that a node must be in a given status for,
+	// after which the node is considered unhealthy.
+	// For example, with a value of "1h", the node must match the status
+	// for at least 1 hour before being considered unhealthy.
+	// +required
+	Timeout metav1.Duration `json:"timeout"`
+}
+
+// ANCHOR_END: UnhealthyMachineCondition
+
 // ANCHOR: MachineHealthCheckStatus
 
 // MachineHealthCheckStatus defines the observed state of MachineHealthCheck.

api/core/v1beta1/zz_generated.openapi.go

@@ -161,6 +161,9 @@ const (
 
 	// UnhealthyNodeConditionV1Beta1Reason is the reason used when a machine's node has one of the MachineHealthCheck's unhealthy conditions.
 	UnhealthyNodeConditionV1Beta1Reason = "UnhealthyNode"
+
+	// UnhealthyMachineConditionV1Beta1Reason is the reason used when a machine has one of the MachineHealthCheck's unhealthy conditions.
+	UnhealthyMachineConditionV1Beta1Reason = "UnhealthyMachine"
 )
 
 const (