Enable health checks on all Kubernetes clusters by default

A per-resource health check config approach is implemented for enabling ease of adoption of new health checks, while avoiding migration of the backend database.

Changes:
- Added `default_kube` health check config which enables health checks on all Kubernetes clusters
- Revised initialization and insert logic for health check configs

Part of #58413

Author: rana

constants.go

@@ -775,9 +775,16 @@ const (
 var PresetRoles = []string{PresetEditorRoleName, PresetAccessRoleName, PresetAuditorRoleName}
 
 const (
-	// PresetDefaultHealthCheckConfigName is the name of a preset
-	// default health_check_config that enables health checks for all resources.
-	PresetDefaultHealthCheckConfigName = "default"
+	// PresetDefaultHealthCheckConfigDBName is the name of a preset
+	// health_check_config that enables health checks for all
+	// database resources. For historical reasons, this preset is named
+	// "default" even though it applies only to databases.
+	PresetDefaultHealthCheckConfigDBName = "default"
+
+	// PresetDefaultHealthCheckConfigKubeName is the name of a preset
+	// health_check_config that enables health checks for all
+	// Kubernetes resources.
+	PresetDefaultHealthCheckConfigKubeName = "default_kube"
 )
 
 const (

lib/auth/init.go

@@ -1489,21 +1489,57 @@ func createPresetDatabaseObjectImportRule(ctx context.Context, rules services.Da
 // createPresetHealthCheckConfig creates a default preset health check config
 // resource that enables health checks on all resources.
 func createPresetHealthCheckConfig(ctx context.Context, svc services.HealthCheckConfig) error {
+	// To support developing health checks for multiple resources over time,
+	// while avoiding migration of the backend database,
+	// and enabling ease of health check adoption:
+	// 	- Create a health check preset for each resource (db, kube, etc)
+
 	page, _, err := svc.ListHealthCheckConfigs(ctx, 0, "")
 	if err != nil {
 		return trace.Wrap(err, "failed listing available health check configs")
 	}
-	if len(page) > 0 {
+	if len(page) == 0 {
+		// No health check configs exist.
+		// Create all preset configs.
+		presetDB := services.NewPresetHealthCheckConfigDB()
+		_, err = svc.CreateHealthCheckConfig(ctx, presetDB)
+		if err != nil && !trace.IsAlreadyExists(err) {
+			return trace.Wrap(err,
+				"failed creating preset health_check_config %s",
+				presetDB.GetMetadata().GetName(),
+			)
+		}
+		presetKube := services.NewPresetHealthCheckConfigKube()
+		_, err = svc.CreateHealthCheckConfig(ctx, presetKube)
+		if err != nil && !trace.IsAlreadyExists(err) {
+			return trace.Wrap(err,
+				"failed creating preset health_check_config %s",
+				presetKube.GetMetadata().GetName(),
+			)
+		}
 		return nil
+	} else {
+		// Health check configs exist.
+		// Create per-resource presets.
+		// Skip creating a DB preset; historically, it's the first, and already exists.
+
+		// Look for an existing kube preset.
+		for _, cfg := range page {
+			if cfg.GetMetadata().GetName() == teleport.PresetDefaultHealthCheckConfigKubeName {
+				return nil
+			}
+		}
+		// Create a kube preset.
+		presetKube := services.NewPresetHealthCheckConfigKube()
+		_, err = svc.CreateHealthCheckConfig(ctx, presetKube)
+		if err != nil && !trace.IsAlreadyExists(err) {
+			return trace.Wrap(err,
+				"failed creating preset health_check_config %s",
+				presetKube.GetMetadata().GetName(),
+			)
+		}
 	}
-	preset := services.NewPresetHealthCheckConfig()
-	_, err = svc.CreateHealthCheckConfig(ctx, preset)
-	if err != nil && !trace.IsAlreadyExists(err) {
-		return trace.Wrap(err,
-			"failed creating preset health_check_config %s",
-			preset.GetMetadata().GetName(),
-		)
-	}
+
 	return nil
 }
 

lib/auth/init_test.go

@@ -985,7 +985,7 @@ func TestPresets(t *testing.T) {
 			require.NoError(t, err)
 		}
 
-		cfg, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigName)
+		cfg, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigDBName)
 		require.NoError(t, err)
 		require.NotNil(t, cfg)
 	})
@@ -1021,7 +1021,7 @@ func TestPresets(t *testing.T) {
 		as.SetClock(clock)
 
 		// an existing health check config should not be modified by init
-		cfg := services.NewPresetHealthCheckConfig()
+		cfg := services.NewPresetHealthCheckConfigDB()
 		cfg.Spec.Interval = durationpb.New(42 * time.Second)
 		cfg, err := as.CreateHealthCheckConfig(ctx, cfg)
 		require.NoError(t, err)

lib/services/presets.go

@@ -855,15 +855,15 @@ func NewPresetMCPUserRole() types.Role {
 	return role
 }
 
-// NewPresetHealthCheckConfig returns a preset default health_check_config that
-// enables health checks for all resources.
-func NewPresetHealthCheckConfig() *healthcheckconfigv1.HealthCheckConfig {
+// NewPresetHealthCheckConfigDB returns a preset health_check_config
+// that enables health checks for all databases resources.
+func NewPresetHealthCheckConfigDB() *healthcheckconfigv1.HealthCheckConfig {
 	return &healthcheckconfigv1.HealthCheckConfig{
 		Kind:    types.KindHealthCheckConfig,
 		Version: types.V1,
 		Metadata: &headerv1.Metadata{
-			Name:        teleport.PresetDefaultHealthCheckConfigName,
-			Description: "Enables all health checks by default",
+			Name:        teleport.PresetDefaultHealthCheckConfigDBName,
+			Description: "Enables health checks for all databases by default",
 			Namespace:   apidefaults.Namespace,
 			Labels: map[string]string{
 				types.TeleportInternalResourceType: types.PresetResource,
@@ -881,6 +881,32 @@ func NewPresetHealthCheckConfig() *healthcheckconfigv1.HealthCheckConfig {
 	}
 }
 
+// NewPresetHealthCheckConfigKube returns a preset health_check_config
+// that enables health checks for all Kubernetes resources.
+func NewPresetHealthCheckConfigKube() *healthcheckconfigv1.HealthCheckConfig {
+	return &healthcheckconfigv1.HealthCheckConfig{
+		Kind:    types.KindHealthCheckConfig,
+		Version: types.V1,
+		Metadata: &headerv1.Metadata{
+			Name:        teleport.PresetDefaultHealthCheckConfigKubeName,
+			Description: "Enables health checks for all Kubernetes clusters by default",
+			Namespace:   apidefaults.Namespace,
+			Labels: map[string]string{
+				types.TeleportInternalResourceType: types.PresetResource,
+			},
+		},
+		Spec: &healthcheckconfigv1.HealthCheckConfigSpec{
+			Match: &healthcheckconfigv1.Matcher{
+				// match all kubernetes clusters
+				KubernetesLabels: []*labelv1.Label{{
+					Name:   types.Wildcard,
+					Values: []string{types.Wildcard},
+				}},
+			},
+		},
+	}
+}
+
 // bootstrapRoleMetadataLabels are metadata labels that will be applied to each role.
 // These are intended to add labels for older roles that didn't previously have them.
 func bootstrapRoleMetadataLabels() map[string]map[string]string {