Enable health checks on all Kubernetes clusters by default

constants.go

@@ -775,9 +775,16 @@ const (
 var PresetRoles = []string{PresetEditorRoleName, PresetAccessRoleName, PresetAuditorRoleName}
 
 const (
-	// PresetDefaultHealthCheckConfigName is the name of a preset
-	// default health_check_config that enables health checks for all resources.
-	PresetDefaultHealthCheckConfigName = "default"
+	// PresetDefaultHealthCheckConfigDBName is the name of a preset
+	// health_check_config that enables health checks for all
+	// database resources. For historical reasons, this preset is named
+	// "default" even though it applies only to databases.
+	PresetDefaultHealthCheckConfigDBName = "default"
+
+	// PresetDefaultHealthCheckConfigKubeName is the name of a preset
+	// health_check_config that enables health checks for all
+	// Kubernetes resources.
+	PresetDefaultHealthCheckConfigKubeName = "default_kube"
 )
 
 const (

lib/auth/export_test.go

@@ -293,8 +293,8 @@ func CreatePresetRoles(ctx context.Context, um PresetRoleManager) error {
 	return createPresetRoles(ctx, um)
 }
 
-func CreatePresetHealthCheckConfig(ctx context.Context, svc services.HealthCheckConfig) error {
-	return createPresetHealthCheckConfig(ctx, svc)
+func CreatePresetHealthCheckConfigs(ctx context.Context, svc services.HealthCheckConfig) error {
+	return createPresetHealthCheckConfigs(ctx, svc)
 }
 
 func GetPresetUsers() []types.User {

lib/auth/init.go

@@ -44,6 +44,7 @@ import (
 	"github.com/gravitational/teleport/api/client/proto"
 	autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1"
 	clusterconfigpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/clusterconfig/v1"
+	healthcheckconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/healthcheckconfig/v1"
 	machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/types/clusterconfig"
@@ -64,6 +65,7 @@ import (
 	"github.com/gravitational/teleport/lib/backend"
 	"github.com/gravitational/teleport/lib/cryptosuites"
 	"github.com/gravitational/teleport/lib/events"
+	"github.com/gravitational/teleport/lib/itertools/stream"
 	"github.com/gravitational/teleport/lib/modules"
 	"github.com/gravitational/teleport/lib/service/servicecfg"
 	"github.com/gravitational/teleport/lib/services"
@@ -680,11 +682,11 @@ func initCluster(ctx context.Context, cfg InitConfig, asrv *Server) error {
 		}
 		span.AddEvent("completed creating database object import rules")
 
-		span.AddEvent("creating preset health check config")
-		if err := createPresetHealthCheckConfig(ctx, asrv); err != nil {
+		span.AddEvent("creating preset health check configs")
+		if err := createPresetHealthCheckConfigs(ctx, asrv); err != nil {
 			return trace.Wrap(err)
 		}
-		span.AddEvent("completed creating preset health check config")
+		span.AddEvent("completed creating preset health check configs")
 	} else {
 		asrv.logger.InfoContext(ctx, "skipping preset role and user creation")
 	}
@@ -1486,23 +1488,40 @@ func createPresetDatabaseObjectImportRule(ctx context.Context, rules services.Da
 	return nil
 }
 
-// createPresetHealthCheckConfig creates a default preset health check config
-// resource that enables health checks on all resources.
-func createPresetHealthCheckConfig(ctx context.Context, svc services.HealthCheckConfig) error {
-	page, _, err := svc.ListHealthCheckConfigs(ctx, 0, "")
+// NewPresetHealthFunc is a function that creates a new HealthCheckConfig preset.
+type NewPresetHealthFunc func() *healthcheckconfigv1.HealthCheckConfig
+
+// newHealthPresets maps preset health names to preset creator functions.
+var newHealthPresets = map[string]NewPresetHealthFunc{
+	teleport.PresetDefaultHealthCheckConfigDBName:   services.NewPresetHealthCheckConfigDB,
+	teleport.PresetDefaultHealthCheckConfigKubeName: services.NewPresetHealthCheckConfigKube,
+}
+
+// createPresetHealthCheckConfigs creates a preset health check config
+// for each resource using the healthcheck package.
+func createPresetHealthCheckConfigs(ctx context.Context, svc services.HealthCheckConfig) error {
+	// The choice to create a preset per-resource is motivated by:
+	// - Supporting existing Teleport clusters already using health checks with some resources
+	// - Avoiding migration of the backend database, which avoids downtime and headaches
+	// - Easing the adoption of health checks for new resources as they are developed over time
+	exists := make(map[string]bool)
+	cfgs, err := stream.Collect(clientutils.Resources(ctx, svc.ListHealthCheckConfigs))
 	if err != nil {
-		return trace.Wrap(err, "failed listing available health check configs")
+		return trace.Wrap(err, "unable to list health check configs")
 	}
-	if len(page) > 0 {
-		return nil
+	for _, cfg := range cfgs {
+		exists[cfg.GetMetadata().GetName()] = true
 	}
-	preset := services.NewPresetHealthCheckConfig()
-	_, err = svc.CreateHealthCheckConfig(ctx, preset)
-	if err != nil && !trace.IsAlreadyExists(err) {
-		return trace.Wrap(err,
-			"failed creating preset health_check_config %s",
-			preset.GetMetadata().GetName(),
-		)
+	var errs []error
+	for name, newPreset := range newHealthPresets {
+		if !exists[name] {
+			if _, err = svc.CreateHealthCheckConfig(ctx, newPreset()); err != nil && !trace.IsAlreadyExists(err) {
+				errs = append(errs, err)
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return trace.NewAggregate(errs...)
 	}
 	return nil
 }

lib/auth/init_test.go

@@ -141,7 +141,7 @@ func TestBadIdentity(t *testing.T) {
 
 	// bad cert type
 	_, err = state.ReadSSHIdentityFromKeyPair(priv, pub)
-	require.IsType(t, trace.BadParameter(""), err)
+	require.ErrorIs(t, trace.BadParameter("failed to parse server certificate: not an SSH certificate"), err)
 
 	// missing authority domain
 	cert, err := a.GenerateHostCert(sshca.HostCertificateRequest{
@@ -158,7 +158,7 @@ func TestBadIdentity(t *testing.T) {
 	require.NoError(t, err)
 
 	_, err = state.ReadSSHIdentityFromKeyPair(priv, cert)
-	require.IsType(t, trace.BadParameter(""), err)
+	require.ErrorIs(t, trace.BadParameter("missing cert extension x-teleport-authority"), err)
 
 	// missing host uuid
 	cert, err = a.GenerateHostCert(sshca.HostCertificateRequest{
@@ -175,7 +175,7 @@ func TestBadIdentity(t *testing.T) {
 	require.NoError(t, err)
 
 	_, err = state.ReadSSHIdentityFromKeyPair(priv, cert)
-	require.IsType(t, trace.BadParameter(""), err)
+	require.ErrorIs(t, trace.BadParameter("missing cert extension x-teleport-authority"), err)
 
 	// unrecognized role
 	cert, err = a.GenerateHostCert(sshca.HostCertificateRequest{
@@ -192,7 +192,7 @@ func TestBadIdentity(t *testing.T) {
 	require.NoError(t, err)
 
 	_, err = state.ReadSSHIdentityFromKeyPair(priv, cert)
-	require.IsType(t, trace.BadParameter(""), err)
+	require.ErrorIs(t, trace.BadParameter("invalid role \"bad role\""), err)
 }
 
 func TestSignatureAlgorithmSuite(t *testing.T) {
@@ -969,14 +969,14 @@ func TestPresets(t *testing.T) {
 		err := auth.CreatePresetRoles(ctx, as)
 		require.NoError(t, err)
 
-		err = auth.CreatePresetHealthCheckConfig(ctx, as)
+		err = auth.CreatePresetHealthCheckConfigs(ctx, as)
 		require.NoError(t, err)
 
 		// Second call should not fail
 		err = auth.CreatePresetRoles(ctx, as)
 		require.NoError(t, err)
 
-		err = auth.CreatePresetHealthCheckConfig(ctx, as)
+		err = auth.CreatePresetHealthCheckConfigs(ctx, as)
 		require.NoError(t, err)
 
 		// Presets were created
@@ -985,7 +985,7 @@ func TestPresets(t *testing.T) {
 			require.NoError(t, err)
 		}
 
-		cfg, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigName)
+		cfg, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigDBName)
 		require.NoError(t, err)
 		require.NotNil(t, cfg)
 	})
@@ -1021,12 +1021,12 @@ func TestPresets(t *testing.T) {
 		as.SetClock(clock)
 
 		// an existing health check config should not be modified by init
-		cfg := services.NewPresetHealthCheckConfig()
+		cfg := services.NewPresetHealthCheckConfigDB()
 		cfg.Spec.Interval = durationpb.New(42 * time.Second)
 		cfg, err := as.CreateHealthCheckConfig(ctx, cfg)
 		require.NoError(t, err)
 
-		err = auth.CreatePresetHealthCheckConfig(ctx, as)
+		err = auth.CreatePresetHealthCheckConfigs(ctx, as)
 		require.NoError(t, err)
 
 		// Preset was created. Ensure it didn't overwrite the existing config
@@ -1035,6 +1035,53 @@ func TestPresets(t *testing.T) {
 		require.Equal(t, cfg.Spec.Interval.AsDuration(), got.Spec.Interval.AsDuration())
 	})
 
+	t.Run("AddAllHealthCheckConfigs", func(t *testing.T) {
+		as := newTestAuthServer(ctx, t)
+		clock := clockwork.NewFakeClock()
+		as.SetClock(clock)
+
+		// Create all health check presets.
+		err := auth.CreatePresetHealthCheckConfigs(ctx, as)
+		require.NoError(t, err)
+
+		// Check that all presets were created.
+		db, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigDBName)
+		require.NoError(t, err)
+		require.NotNil(t, db)
+		require.Equal(t,
+			teleport.PresetDefaultHealthCheckConfigDBName,
+			db.GetMetadata().GetName())
+		kube, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigKubeName)
+		require.NoError(t, err)
+		require.NotNil(t, kube)
+		require.Equal(t,
+			teleport.PresetDefaultHealthCheckConfigKubeName,
+			kube.GetMetadata().GetName())
+	})
+
+	t.Run("AddKubeHealthCheckConfig", func(t *testing.T) {
+		as := newTestAuthServer(ctx, t)
+		clock := clockwork.NewFakeClock()
+		as.SetClock(clock)
+
+		// Simulate an existing cluster with db health checks.
+		db, err := as.CreateHealthCheckConfig(ctx, services.NewPresetHealthCheckConfigDB())
+		require.NoError(t, err)
+		require.NotNil(t, db)
+
+		// Attempt to create all health check presets.
+		err = auth.CreatePresetHealthCheckConfigs(ctx, as)
+		require.NoError(t, err)
+
+		// Check that the kube preset was created.
+		kube, err := as.GetHealthCheckConfig(ctx, teleport.PresetDefaultHealthCheckConfigKubeName)
+		require.NoError(t, err)
+		require.NotNil(t, kube)
+		require.Equal(t,
+			teleport.PresetDefaultHealthCheckConfigKubeName,
+			kube.GetMetadata().GetName())
+	})
+
 	// If a default allow condition is not present, ensure it gets added.
 	t.Run("AddDefaultAllowConditions", func(t *testing.T) {
 		as := newTestAuthServer(ctx, t)

lib/services/presets.go

@@ -855,15 +855,15 @@ func NewPresetMCPUserRole() types.Role {
 	return role
 }
 
-// NewPresetHealthCheckConfig returns a preset default health_check_config that
-// enables health checks for all resources.
-func NewPresetHealthCheckConfig() *healthcheckconfigv1.HealthCheckConfig {
+// NewPresetHealthCheckConfigDB returns a preset health_check_config
+// enabling health checks for all databases resources.
+func NewPresetHealthCheckConfigDB() *healthcheckconfigv1.HealthCheckConfig {
 	return &healthcheckconfigv1.HealthCheckConfig{
 		Kind:    types.KindHealthCheckConfig,
 		Version: types.V1,
 		Metadata: &headerv1.Metadata{
-			Name:        teleport.PresetDefaultHealthCheckConfigName,
-			Description: "Enables all health checks by default",
+			Name:        teleport.PresetDefaultHealthCheckConfigDBName,
+			Description: "Enables health checks for all databases by default",
 			Namespace:   apidefaults.Namespace,
 			Labels: map[string]string{
 				types.TeleportInternalResourceType: types.PresetResource,
@@ -881,6 +881,32 @@ func NewPresetHealthCheckConfig() *healthcheckconfigv1.HealthCheckConfig {
 	}
 }
 
+// NewPresetHealthCheckConfigKube returns a preset health_check_config
+// enabling health checks for all Kubernetes resources.
+func NewPresetHealthCheckConfigKube() *healthcheckconfigv1.HealthCheckConfig {
+	return &healthcheckconfigv1.HealthCheckConfig{
+		Kind:    types.KindHealthCheckConfig,
+		Version: types.V1,
+		Metadata: &headerv1.Metadata{
+			Name:        teleport.PresetDefaultHealthCheckConfigKubeName,
+			Description: "Enables health checks for all Kubernetes clusters by default",
+			Namespace:   apidefaults.Namespace,
+			Labels: map[string]string{
+				types.TeleportInternalResourceType: types.PresetResource,
+			},
+		},
+		Spec: &healthcheckconfigv1.HealthCheckConfigSpec{
+			Match: &healthcheckconfigv1.Matcher{
+				// match all kubernetes clusters
+				KubernetesLabels: []*labelv1.Label{{
+					Name:   types.Wildcard,
+					Values: []string{types.Wildcard},
+				}},
+			},
+		},
+	}
+}
+
 // bootstrapRoleMetadataLabels are metadata labels that will be applied to each role.
 // These are intended to add labels for older roles that didn't previously have them.
 func bootstrapRoleMetadataLabels() map[string]map[string]string {