feat: Introduce Dedicated Istio Gateway Secret

.github/actions/deploy-lifecycle-manager-e2e/action.yml

@@ -81,7 +81,8 @@ runs:
         kustomize edit add patch --path requeue-interval-patch.yaml --kind Deployment
         popd
     - name: Patch CA certificate renewBefore
-      if: ${{matrix.e2e-test == 'ca-certificate-rotation'}}
+      if: ${{matrix.e2e-test == 'ca-certificate-rotation' ||
+        matrix.e2e-test == 'istio-gateway-secret-rotation'}}
       working-directory: lifecycle-manager
       shell: bash
       run: |

.github/workflows/test-e2e-with-modulereleasemeta.yml

@@ -49,6 +49,7 @@ jobs:
           - unmanage-module
           - skip-manifest-reconciliation
           - ca-certificate-rotation
+          - istio-gateway-secret-rotation
           - self-signed-certificate-rotation
           - mandatory-module
           - mandatory-module-metrics

.github/workflows/test-e2e.yml

@@ -50,6 +50,7 @@ jobs:
           - module-install-by-version
           - skip-manifest-reconciliation
           - ca-certificate-rotation
+          - istio-gateway-secret-rotation
           - self-signed-certificate-rotation
           - mandatory-module
           - mandatory-module-metrics

cmd/main.go

@@ -36,7 +36,9 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	machineryruntime "k8s.io/apimachinery/pkg/runtime"
 	machineryutilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
 	k8sclientscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -61,6 +63,7 @@ import (
 	"github.com/kyma-project/lifecycle-manager/internal/pkg/flags"
 	"github.com/kyma-project/lifecycle-manager/internal/pkg/metrics"
 	"github.com/kyma-project/lifecycle-manager/internal/remote"
+	"github.com/kyma-project/lifecycle-manager/pkg/gatewaysecret"
 	"github.com/kyma-project/lifecycle-manager/pkg/log"
 	"github.com/kyma-project/lifecycle-manager/pkg/matcher"
 	"github.com/kyma-project/lifecycle-manager/pkg/queue"
@@ -201,12 +204,21 @@ func setupManager(flagVar *flags.FlagVar, cacheOptions cache.Options, scheme *ma
 	go cleanupStoredVersions(flagVar.DropCrdStoredVersionMap, mgr, setupLog)
 	go scheduleMetricsCleanup(kymaMetrics, flagVar.MetricsCleanupIntervalInMinutes, mgr, setupLog)
 
+	go setupIstioGatewaySecretRotation(config, kcpClient, setupLog)
+
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(runtimeProblemExitCode)
 	}
 }
 
+func setupIstioGatewaySecretRotation(config *rest.Config, kcpClient *remote.ConfigAndClient, setupLog logr.Logger) {
+	kcpClientset := kubernetes.NewForConfigOrDie(config)
+	gatewaySecretHandler := gatewaysecret.NewGatewaySecretHandler(kcpClient)
+
+	gatewaysecret.StartRootCertificateWatch(kcpClientset, gatewaySecretHandler, setupLog)
+}
+
 func addHealthChecks(mgr manager.Manager, setupLog logr.Logger) {
 	// +kubebuilder:scaffold:builder
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -319,7 +331,6 @@ func setupKymaReconciler(mgr ctrl.Manager, descriptorProvider *provider.CachedDe
 func createSkrWebhookManager(mgr ctrl.Manager, skrContextFactory remote.SkrContextProvider,
 	flagVar *flags.FlagVar,
 ) (*watcher.SKRWebhookManifestManager, error) {
-	caCertificateCache := watcher.NewCACertificateCache(flagVar.CaCertCacheTTL)
 	config := watcher.SkrWebhookManagerConfig{
 		SKRWatcherPath:         flagVar.WatcherResourcesPath,
 		SkrWatcherImage:        flagVar.GetWatcherImage(),
@@ -349,7 +360,6 @@ func createSkrWebhookManager(mgr ctrl.Manager, skrContextFactory remote.SkrConte
 	return watcher.NewSKRWebhookManifestManager(
 		mgr.GetClient(),
 		skrContextFactory,
-		caCertificateCache,
 		config,
 		certConfig,
 		resolvedKcpAddr)

config/manager/kustomization.yaml

@@ -10,5 +10,5 @@ generatorOptions:
 
 images:
 - name: controller
-  newName: europe-docker.pkg.dev/kyma-project/prod/lifecycle-manager
-  newTag: latest
+  newName: k3d-kcp-registry.localhost:5000/lifecycle-manager
+  newTag: "20241114183639"

config/rbac/namespace_bindings/watcher_certmanager_role.yaml

@@ -15,6 +15,7 @@ rules:
     - watch
     - create
     - delete
+    - update
 - apiGroups:
     - cert-manager.io
   resources:

config/watcher/gateway.yaml

@@ -20,5 +20,5 @@ spec:
         number: 443
         protocol: HTTPS
       tls:
-        credentialName: klm-watcher
+        credentialName: klm-istio-gateway
         mode: MUTUAL

internal/pkg/flags/flags.go

@@ -44,7 +44,6 @@ const (
 	DefaultIstioGatewayNamespace                                        = "kcp-system"
 	DefaultIstioNamespace                                               = "istio-system"
 	DefaultCaCertName                                                   = "klm-watcher-serving"
-	DefaultCaCertCacheTTL                                 time.Duration = 1 * time.Hour
 	DefaultSelfSignedCertDuration                         time.Duration = 90 * 24 * time.Hour
 	DefaultSelfSignedCertRenewBefore                      time.Duration = 60 * 24 * time.Hour
 	DefaultSelfSignedCertificateRenewBuffer                             = 24 * time.Hour
@@ -204,8 +203,6 @@ func DefineFlagVar() *FlagVar {
 		"Name of the namespace for syncing remote Kyma and module catalog")
 	flag.StringVar(&flagVar.CaCertName, "ca-cert-name", DefaultCaCertName,
 		"Name of the CA Certificate in Istio Namespace which is used to sign SKR Certificates")
-	flag.DurationVar(&flagVar.CaCertCacheTTL, "ca-cert-cache-ttl", DefaultCaCertCacheTTL,
-		"The ttl for the CA Certificate Cache")
 	flag.DurationVar(&flagVar.SelfSignedCertDuration, "self-signed-cert-duration", DefaultSelfSignedCertDuration,
 		"The lifetime duration of self-signed certificate, minimum accepted duration is 1 hour.")
 	flag.DurationVar(&flagVar.SelfSignedCertRenewBefore, "self-signed-cert-renew-before",
@@ -288,7 +285,6 @@ type FlagVar struct {
 	SkipPurgingFor                         string
 	RemoteSyncNamespace                    string
 	CaCertName                             string
-	CaCertCacheTTL                         time.Duration
 	IsKymaManaged                          bool
 	SelfSignedCertDuration                 time.Duration
 	SelfSignedCertRenewBefore              time.Duration

internal/pkg/flags/flags_test.go

@@ -175,11 +175,6 @@ func Test_ConstantFlags(t *testing.T) {
 			constValue:    DefaultCaCertName,
 			expectedValue: "klm-watcher-serving",
 		},
-		{
-			constName:     "DefaultCaCertCacheTTL",
-			constValue:    DefaultCaCertCacheTTL.String(),
-			expectedValue: (1 * time.Hour).String(),
-		},
 		{
 			constName:     "DefaultSelfSignedCertDuration",
 			constValue:    DefaultSelfSignedCertDuration.String(),

pkg/gatewaysecret/handler.go

@@ -0,0 +1,203 @@
+package gatewaysecret
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+	"github.com/go-logr/logr"
+	apicorev1 "k8s.io/api/core/v1"
+	apimetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kyma-project/lifecycle-manager/pkg/util"
+)
+
+const (
+	LastModifiedAtAnnotation = "lastModifiedAt"
+	gatewaySecretName        = "klm-istio-gateway" //nolint:gosec // gatewaySecretName is not a credential
+	kcpRootSecretName        = "klm-watcher"
+	kcpCACertName            = "klm-watcher-serving"
+	istioNamespace           = "istio-system"
+)
+
+var errCouldNotGetLastModifiedAt = errors.New("getting lastModifiedAt time failed")
+
+type GatewaySecretHandler struct {
+	kcpClient client.Client
+}
+
+func NewGatewaySecretHandler(kcpClient client.Client) *GatewaySecretHandler {
+	return &GatewaySecretHandler{
+		kcpClient: kcpClient,
+	}
+}
+
+func (gsh *GatewaySecretHandler) ManageGatewaySecret(ctx context.Context, rootSecret *apicorev1.Secret) error {
+	gwSecret, err := gsh.FindGatewaySecret(ctx)
+
+	if util.IsNotFound(err) {
+		return gsh.handleNonExisting(ctx, rootSecret)
+	}
+	if err != nil {
+		return err
+	}
+
+	return gsh.handleExisting(ctx, rootSecret, gwSecret)
+}
+
+func (gsh *GatewaySecretHandler) handleNonExisting(ctx context.Context, rootSecret *apicorev1.Secret) error {
+	gwSecret := NewGatewaySecret(rootSecret)
+	return gsh.Create(ctx, gwSecret)
+}
+
+func (gsh *GatewaySecretHandler) handleExisting(ctx context.Context,
+	rootSecret *apicorev1.Secret, gwSecret *apicorev1.Secret,
+) error {
+	caCert, err := gsh.GetRootCACertificate(ctx)
+	if err != nil {
+		return err
+	}
+	if !GatewaySecretRequiresUpdate(gwSecret, caCert) {
+		return nil
+	}
+	CopyRootSecretDataIntoGatewaySecret(gwSecret, rootSecret)
+	return gsh.Update(ctx, gwSecret)
+}
+
+func CopyRootSecretDataIntoGatewaySecret(gwSecret *apicorev1.Secret, rootSecret *apicorev1.Secret) {
+	gwSecret.Data["tls.crt"] = rootSecret.Data["tls.crt"]
+	gwSecret.Data["tls.key"] = rootSecret.Data["tls.key"]
+	gwSecret.Data["ca.crt"] = rootSecret.Data["ca.crt"]
+}
+
+func GatewaySecretRequiresUpdate(gwSecret *apicorev1.Secret, caCert certmanagerv1.Certificate) bool {
+	if gwSecretLastModifiedAt, err := GetValidLastModifiedAt(gwSecret); err == nil {
+		if caCert.Status.NotBefore != nil && gwSecretLastModifiedAt.After(caCert.Status.NotBefore.Time) {
+			return false
+		}
+	}
+	return true
+}
+
+func GetValidLastModifiedAt(secret *apicorev1.Secret) (time.Time, error) {
+	if gwSecretLastModifiedAtValue, ok := secret.Annotations[LastModifiedAtAnnotation]; ok {
+		if gwSecretLastModifiedAt, err := time.Parse(time.RFC3339, gwSecretLastModifiedAtValue); err == nil {
+			return gwSecretLastModifiedAt, nil
+		}
+	}
+	return time.Time{}, errCouldNotGetLastModifiedAt
+}
+
+func (gsh *GatewaySecretHandler) FindGatewaySecret(ctx context.Context) (*apicorev1.Secret, error) {
+	return GetGatewaySecret(ctx, gsh.kcpClient)
+}
+
+func (gsh *GatewaySecretHandler) Create(ctx context.Context, secret *apicorev1.Secret) error {
+	gsh.updateLastModifiedAt(secret)
+	if err := gsh.kcpClient.Create(ctx, secret); err != nil {
+		return fmt.Errorf("failed to create secret %s: %w", secret.Name, err)
+	}
+	return nil
+}
+
+func (gsh *GatewaySecretHandler) Update(ctx context.Context, secret *apicorev1.Secret) error {
+	gsh.updateLastModifiedAt(secret)
+	if err := gsh.kcpClient.Update(ctx, secret); err != nil {
+		return fmt.Errorf("failed to update secret %s: %w", secret.Name, err)
+	}
+	return nil
+}
+
+func (gsh *GatewaySecretHandler) GetRootCACertificate(ctx context.Context) (certmanagerv1.Certificate, error) {
+	caCert := certmanagerv1.Certificate{}
+	if err := gsh.kcpClient.Get(ctx,
+		client.ObjectKey{Namespace: istioNamespace, Name: kcpCACertName},
+		&caCert); err != nil {
+		return certmanagerv1.Certificate{}, fmt.Errorf("failed to get CA certificate: %w", err)
+	}
+	return caCert, nil
+}
+
+func (gsh *GatewaySecretHandler) updateLastModifiedAt(secret *apicorev1.Secret) {
+	if secret.Annotations == nil {
+		secret.Annotations = make(map[string]string)
+	}
+	secret.Annotations[LastModifiedAtAnnotation] = apimetav1.Now().Format(time.RFC3339)
+}
+
+func NewGatewaySecret(rootSecret *apicorev1.Secret) *apicorev1.Secret {
+	gwSecret := &apicorev1.Secret{
+		TypeMeta: apimetav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: apicorev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: apimetav1.ObjectMeta{
+			Name:      gatewaySecretName,
+			Namespace: istioNamespace,
+		},
+		Data: map[string][]byte{
+			"tls.crt": rootSecret.Data["tls.crt"],
+			"tls.key": rootSecret.Data["tls.key"],
+			"ca.crt":  rootSecret.Data["ca.crt"],
+		},
+	}
+	return gwSecret
+}
+
+func GetGatewaySecret(ctx context.Context, clnt client.Client) (*apicorev1.Secret, error) {
+	secret := &apicorev1.Secret{}
+	if err := clnt.Get(ctx, client.ObjectKey{
+		Name:      gatewaySecretName,
+		Namespace: istioNamespace,
+	}, secret); err != nil {
+		return nil, fmt.Errorf("failed to get secret %s: %w", gatewaySecretName, err)
+	}
+	return secret, nil
+}
+
+func StartRootCertificateWatch(clientset *kubernetes.Clientset, gatewaySecretHandler *GatewaySecretHandler,
+	log logr.Logger,
+) {
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+
+	secretWatch, err := clientset.CoreV1().Secrets(istioNamespace).Watch(ctx, apimetav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector(apimetav1.ObjectNameField, kcpRootSecretName).String(),
+	})
+	if err != nil {
+		log.Error(err, "unable to start watching root certificate")
+		panic(err)
+	}
+
+	WatchEvents(ctx, secretWatch.ResultChan(), gatewaySecretHandler.ManageGatewaySecret, log)
+}
+
+func WatchEvents(ctx context.Context, watchEvents <-chan watch.Event,
+	manageGatewaySecretFunc func(context.Context, *apicorev1.Secret) error, log logr.Logger,
+) {
+	for event := range watchEvents {
+		rootCASecret, _ := event.Object.(*apicorev1.Secret)
+
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			err := manageGatewaySecretFunc(ctx, rootCASecret)
+			if err != nil {
+				log.Error(err, "unable to manage istio gateway secret")
+			}
+		case watch.Deleted:
+			// ignored because it is an invalid state and cert manager should not delete the root secret
+			// even if it is deleted, the certificate manager will recreate it, and trigger the watch event
+			fallthrough
+		case watch.Error, watch.Bookmark:
+			fallthrough
+		default:
+			continue
+		}
+	}
+}