Kms vault lifecycle

go.mod

@@ -136,3 +136,5 @@ require (
 )
 
 replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1
+
+replace github.com/openshift/library-go => github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86

go.sum

@@ -50,6 +50,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
 github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86 h1:A83Pqt6r8r2v/1I3mARfUS0aD7P3BNEyFAUy1vtcrRg=
+github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -165,8 +167,6 @@ github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+S
 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE=
 github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a h1:EKx2XhOKehd1C5ptY7IrLl4WV35E8kP0pRPnG5BUZXk=
 github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a/go.mod h1:V933kvY/cb/Un7UCEOhXHUySNX327u7Epe8g9KNqg2Q=
-github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9 h1:1ubwPydT+ABjfvmeiv4hoJQ0gIDCyMq/U5UyHrrpefA=
-github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 h1:PMTgifBcBRLJJiM+LgSzPDTk9/Rx4qS09OUrfpY6GBQ=
 github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=

pkg/operator/targetconfigcontroller/targetconfigcontroller.go

@@ -22,7 +22,7 @@ import (
 	"github.com/openshift/library-go/pkg/controller/factory"
 	"github.com/openshift/library-go/pkg/operator/certrotation"
 	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
-	encryptionkms "github.com/openshift/library-go/pkg/operator/encryption/kms"
+	kmspluginlifecycle "github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	"github.com/openshift/library-go/pkg/operator/resource/resourcehelper"
@@ -56,8 +56,9 @@ type TargetConfigController struct {
 
 	operatorClient v1helpers.StaticPodOperatorClient
 
-	kubeClient          kubernetes.Interface
-	configMapLister     corev1listers.ConfigMapLister
+	kubeClient      kubernetes.Interface
+	configMapLister corev1listers.ConfigMapLister
+
 	featureGateAccessor featuregates.FeatureGateAccess
 
 	isStartupMonitorEnabledFn      func() (bool, error)
@@ -224,7 +225,7 @@ func createTargetConfig(ctx context.Context, c TargetConfigController, recorder
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap/config", err))
 	}
-	_, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.featureGateAccessor, c.isStartupMonitorEnabledFn, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec, c.operatorImageVersion)
+	_, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.kubeClient.CoreV1(), c.featureGateAccessor, c.isStartupMonitorEnabledFn, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec, c.operatorImageVersion)
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap/kube-apiserver-pod", err))
 	}
@@ -308,7 +309,7 @@ func manageKubeAPIServerConfig(ctx context.Context, client coreclientv1.ConfigMa
 	return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap)
 }
 
-func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, featureGateAccessor featuregates.FeatureGateAccess, isStartupMonitorEnabledFn func() (bool, error), recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec, operatorImageVersion string) (*corev1.ConfigMap, bool, error) {
+func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, secretClient coreclientv1.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess, isStartupMonitorEnabledFn func() (bool, error), recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec, operatorImageVersion string) (*corev1.ConfigMap, bool, error) {
 	appliedPodTemplate, err := manageTemplate(string(bindata.MustAsset("assets/kube-apiserver/pod.yaml")), imagePullSpec, operatorImagePullSpec, operatorImageVersion, operatorSpec)
 	if err != nil {
 		return nil, false, err
@@ -329,8 +330,8 @@ func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, featu
 		required.Spec.Containers[i].Env = append(container.Env, proxyEnvVars...)
 	}
 
-	if err := encryptionkms.AddKMSPluginVolumeAndMountToPodSpec(&required.Spec, "kube-apiserver", featureGateAccessor); err != nil {
-		return nil, false, fmt.Errorf("failed to add KMS encryption volumes: %w", err)
+	if err := kmspluginlifecycle.AddKMSPluginSidecarToPodSpec(ctx, &required.Spec, "kube-apiserver", operatorclient.TargetNamespace, "encryption-config", secretClient, featureGateAccessor); err != nil {
+		return nil, false, fmt.Errorf("failed to add KMS plugin to pod spec: %w", err)
 	}
 
 	configMap := resourceread.ReadConfigMapV1OrDie(bindata.MustAsset("assets/kube-apiserver/pod-cm.yaml"))

test/e2e-encryption-kms/encryption_kms.go

@@ -5,23 +5,28 @@ import (
 	"fmt"
 	"math/rand/v2"
 	"testing"
+	"time"
 
 	g "github.com/onsi/ginkgo/v2"
 
-	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
 	operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption"
+	libraryapiserver "github.com/openshift/library-go/test/library/apiserver"
 	library "github.com/openshift/library-go/test/library/encryption"
 	librarykms "github.com/openshift/library-go/test/library/encryption/kms"
 )
 
 var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() {
-	g.It("TestKMSEncryptionOnOff [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func() {
-		testKMSEncryptionOnOff(g.GinkgoTB())
+	g.It("TestKMSEncryptionOnOff [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) {
+		testKMSEncryptionOnOff(ctx, g.GinkgoTB())
 	})
 
-	g.It("TestKMSEncryptionProvidersMigration [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func() {
-		testKMSEncryptionProvidersMigration(g.GinkgoTB())
+	g.It("TestKMSEncryptionProvidersMigration [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) {
+		testKMSEncryptionProvidersMigration(ctx, g.GinkgoTB())
+	})
+
+	g.It("TestKMSEncryptionInvalidImageRecovery [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) {
+		testKMSEncryptionInvalidImageRecovery(ctx, g.GinkgoTB())
 	})
 })
 
@@ -37,12 +42,12 @@ var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() {
 // 8. Verifies secret is encrypted again
 // 9. Disables encryption (Identity) again
 // 10. Verifies secret is NOT encrypted again
-func testKMSEncryptionOnOff(t testing.TB) {
+func testKMSEncryptionOnOff(ctx context.Context, t testing.TB) {
 	// Deploy the mock KMS plugin for testing.
 	// NOTE: This manual deployment is only required for KMS v1. In the future,
 	// the platform will manage the KMS plugins, and this code will no longer be needed.
-	librarykms.DeployUpstreamMockKMSPlugin(context.Background(), t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount)
-	library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{
+	librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount)
+	library.TestEncryptionTurnOnAndOff(ctx, t, library.OnOffScenario{
 		BasicScenario: library.BasicScenario{
 			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
@@ -57,10 +62,7 @@ func testKMSEncryptionOnOff(t testing.TB) {
 		AssertResourceNotEncryptedFunc: operatorencryption.AssertSecretOfLifeNotEncrypted,
 		ResourceFunc:                   operatorencryption.SecretOfLife,
 		ResourceName:                   "SecretOfLife",
-		EncryptionProvider: library.EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{
-			Type: configv1.EncryptionTypeKMS,
-			KMS:  librarykms.DefaultFakeKMSPluginConfig,
-		}},
+		EncryptionProvider:             library.EncryptionProvider{APIServerEncryption: librarykms.DefaultFakeKMSPluginConfig},
 	})
 }
 
@@ -72,9 +74,9 @@ func testKMSEncryptionOnOff(t testing.TB) {
 // 4. Shuffles the selected AES provider with KMS to create a randomized migration order
 // 5. Migrates between the providers in the shuffled order
 // 6. Verifies secret is correctly encrypted after each migration
-func testKMSEncryptionProvidersMigration(t testing.TB) {
-	librarykms.DeployUpstreamMockKMSPlugin(context.Background(), t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount)
-	library.TestEncryptionProvidersMigration(t, library.ProvidersMigrationScenario{
+func testKMSEncryptionProvidersMigration(ctx context.Context, t testing.TB) {
+	librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount)
+	library.TestEncryptionProvidersMigration(ctx, t, library.ProvidersMigrationScenario{
 		BasicScenario: library.BasicScenario{
 			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
@@ -90,8 +92,40 @@ func testKMSEncryptionProvidersMigration(t testing.TB) {
 		ResourceFunc:                   operatorencryption.SecretOfLife,
 		ResourceName:                   "SecretOfLife",
 		EncryptionProviders: library.ShuffleEncryptionProviders([]library.EncryptionProvider{
-			{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeKMS, KMS: librarykms.DefaultFakeKMSPluginConfig}},
+			{APIServerEncryption: librarykms.DefaultFakeKMSPluginConfig},
 			library.SupportedStaticEncryptionProviders[rand.IntN(len(library.SupportedStaticEncryptionProviders))],
 		}),
 	})
 }
+
+// testKMSEncryptionInvalidImageRecovery tests that an invalid KMS plugin image
+// causes degradation and that fixing the image restores the cluster.
+func testKMSEncryptionInvalidImageRecovery(ctx context.Context, t testing.TB) {
+	librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount)
+	library.TestEncryptionInvalidImageRecovery(ctx, t, library.InvalidImageRecoveryScenario{
+		BasicScenario: library.BasicScenario{
+			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
+			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
+			EncryptionConfigSecretName:      fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace),
+			EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace,
+			OperatorNamespace:               operatorclient.OperatorNamespace,
+			TargetGRs:                       operatorencryption.DefaultTargetGRs,
+			AssertFunc:                      operatorencryption.AssertSecretsAndConfigMaps,
+		},
+		InvalidImageProvider: librarykms.InvalidImageVaultEncryptionProvider,
+		ValidImageProvider:   librarykms.DefaultVaultEncryptionProvider,
+		WaitForDegraded: func(ctx context.Context, t testing.TB) {
+			t.Helper()
+			cs := library.GetClients(t)
+			library.WaitForPodImagePullBackOff(ctx, t, cs.Kube, operatorclient.TargetNamespace, "app=openshift-kube-apiserver", 15*time.Minute)
+		},
+	})
+
+	// wait for all kube-apiserver pods to stabilize on the same revision after recovery
+	cs := library.GetClients(t)
+	podClient := cs.Kube.CoreV1().Pods(operatorclient.TargetNamespace)
+	err := libraryapiserver.WaitForAPIServerToStabilizeOnTheSameRevision(t, podClient)
+	if err != nil {
+		t.Fatalf("apiserver pods did not stabilize after recovery: %v", err)
+	}
+}

test/e2e-encryption-perf/encryption_perf_test.go

@@ -31,7 +31,7 @@ var provider = flag.String("provider", "aescbc", "encryption provider used by th
 
 func TestPerfEncryption(tt *testing.T) {
 	operatorClient := operatorencryption.GetOperator(tt)
-	library.TestPerfEncryption(tt, library.PerfScenario{
+	library.TestPerfEncryption(tt.Context(), tt, library.PerfScenario{
 		BasicScenario: library.BasicScenario{
 			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,

test/e2e-encryption-rotation/encryption_rotation_test.go

@@ -19,7 +19,7 @@ var provider = flag.String("provider", "aescbc", "encryption provider used by th
 // rotation by setting the "encyrption.Reason" in the operator's configuration
 // file
 func TestEncryptionRotation(t *testing.T) {
-	library.TestEncryptionRotation(t, library.RotationScenario{
+	library.TestEncryptionRotation(t.Context(), t, library.RotationScenario{
 		BasicScenario: library.BasicScenario{
 			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,

test/e2e-encryption/encryption_test.go

@@ -14,7 +14,7 @@ import (
 var provider = flag.String("provider", "aescbc", "encryption provider used by the tests")
 
 func TestEncryptionTypeIdentity(t *testing.T) {
-	library.TestEncryptionTypeIdentity(t, library.BasicScenario{
+	library.TestEncryptionTypeIdentity(t.Context(), t, library.BasicScenario{
 		Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 		LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
 		EncryptionConfigSecretName:      fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace),
@@ -26,7 +26,7 @@ func TestEncryptionTypeIdentity(t *testing.T) {
 }
 
 func TestEncryptionTypeUnset(t *testing.T) {
-	library.TestEncryptionTypeUnset(t, library.BasicScenario{
+	library.TestEncryptionTypeUnset(t.Context(), t, library.BasicScenario{
 		Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 		LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
 		EncryptionConfigSecretName:      fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace),
@@ -38,7 +38,7 @@ func TestEncryptionTypeUnset(t *testing.T) {
 }
 
 func TestEncryptionTurnOnAndOff(t *testing.T) {
-	library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{
+	library.TestEncryptionTurnOnAndOff(t.Context(), t, library.OnOffScenario{
 		BasicScenario: library.BasicScenario{
 			Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 			LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,

test/e2e/encryption.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
@@ -12,13 +13,13 @@ import (
 )
 
 var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() {
-	g.It("[Operator][Serial][Timeout:40m] TestEncryptionTypeAESCBC", func() {
-		testEncryptionTypeAESCBC(g.GinkgoTB())
+	g.It("[Operator][Serial][Timeout:40m] TestEncryptionTypeAESCBC", func(ctx context.Context) {
+		testEncryptionTypeAESCBC(ctx, g.GinkgoTB())
 	})
 })
 
-func testEncryptionTypeAESCBC(t testing.TB) {
-	library.TestEncryptionTypeAESCBC(t, library.BasicScenario{
+func testEncryptionTypeAESCBC(ctx context.Context, t testing.TB) {
+	library.TestEncryptionTypeAESCBC(ctx, t, library.BasicScenario{
 		Namespace:                       operatorclient.GlobalMachineSpecifiedConfigNamespace,
 		LabelSelector:                   "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace,
 		EncryptionConfigSecretName:      fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace),

test/e2e/encryption_test.go

@@ -10,5 +10,5 @@ import (
 // This situation is temporary until we test the new e2e-gcp-operator-serial-ote job.
 // Eventually all tests will be run only as part of the OTE framework.
 func TestEncryptionTypeAESCBC(t *testing.T) {
-	testEncryptionTypeAESCBC(t)
+	testEncryptionTypeAESCBC(t.Context(), t)
 }