From ca80506e2db395b461821f479e800e93b7f123d0 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 6 May 2026 16:12:20 -0400 Subject: [PATCH 1/4] kms: support deploying Vault mock plugin --- .../targetconfigcontroller.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go index e1278460fc..615db4e28a 100644 --- a/pkg/operator/targetconfigcontroller/targetconfigcontroller.go +++ b/pkg/operator/targetconfigcontroller/targetconfigcontroller.go @@ -22,7 +22,7 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/certrotation" "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" - encryptionkms "github.com/openshift/library-go/pkg/operator/encryption/kms" + kmspluginlifecycle "github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/resource/resourcehelper" @@ -56,8 +56,9 @@ type TargetConfigController struct { operatorClient v1helpers.StaticPodOperatorClient - kubeClient kubernetes.Interface - configMapLister corev1listers.ConfigMapLister + kubeClient kubernetes.Interface + configMapLister corev1listers.ConfigMapLister + featureGateAccessor featuregates.FeatureGateAccess isStartupMonitorEnabledFn func() (bool, error) @@ -224,7 +225,7 @@ func createTargetConfig(ctx context.Context, c TargetConfigController, recorder if err != nil { errors = append(errors, fmt.Errorf("%q: %v", "configmap/config", err)) } - _, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.featureGateAccessor, c.isStartupMonitorEnabledFn, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec, c.operatorImageVersion) + _, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.kubeClient.CoreV1(), c.featureGateAccessor, c.isStartupMonitorEnabledFn, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec, c.operatorImageVersion) if err != nil { errors = append(errors, fmt.Errorf("%q: %v", "configmap/kube-apiserver-pod", err)) } @@ -308,7 +309,7 @@ func manageKubeAPIServerConfig(ctx context.Context, client coreclientv1.ConfigMa return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap) } -func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, featureGateAccessor featuregates.FeatureGateAccess, isStartupMonitorEnabledFn func() (bool, error), recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec, operatorImageVersion string) (*corev1.ConfigMap, bool, error) { +func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, secretClient coreclientv1.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess, isStartupMonitorEnabledFn func() (bool, error), recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec, operatorImageVersion string) (*corev1.ConfigMap, bool, error) { appliedPodTemplate, err := manageTemplate(string(bindata.MustAsset("assets/kube-apiserver/pod.yaml")), imagePullSpec, operatorImagePullSpec, operatorImageVersion, operatorSpec) if err != nil { return nil, false, err @@ -329,8 +330,8 @@ func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, featu required.Spec.Containers[i].Env = append(container.Env, proxyEnvVars...) } - if err := encryptionkms.AddKMSPluginVolumeAndMountToPodSpec(&required.Spec, "kube-apiserver", featureGateAccessor); err != nil { - return nil, false, fmt.Errorf("failed to add KMS encryption volumes: %w", err) + if err := kmspluginlifecycle.AddKMSPluginSidecarToPodSpec(ctx, &required.Spec, "kube-apiserver", operatorclient.TargetNamespace, "encryption-config", secretClient, featureGateAccessor); err != nil { + return nil, false, fmt.Errorf("failed to add KMS plugin to pod spec: %w", err) } configMap := resourceread.ReadConfigMapV1OrDie(bindata.MustAsset("assets/kube-apiserver/pod-cm.yaml")) From 570a71a12f62456ddcaf50e155f71a6468484c0c Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 20 May 2026 11:39:46 -0400 Subject: [PATCH 2/4] bump(openshift/library-go): to get KMS plugin lifecycle changes --- .../encryption/kms/pluginlifecycle/sidecar.go | 184 ++++++++++++++++++ .../encryption/kms/pluginlifecycle/vault.go | 77 ++++++++ vendor/modules.txt | 1 + 3 files changed, 262 insertions(+) create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go new file mode 100644 index 0000000000..1925cec3e0 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go @@ -0,0 +1,184 @@ +package pluginlifecycle + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" + "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" + "github.com/openshift/library-go/pkg/operator/encryption/encryptiondata" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog/v2" +) + +// sidecarProvider abstracts the construction of a KMS plugin sidecar container for a specific provider (e.g. Vault). +type sidecarProvider interface { + // Name returns the identifier used to name the sidecar container and locate its volume mounts. + Name() string + // BuildSidecarContainer returns a fully configured sidecar container ready to be injected into the API server pod + BuildSidecarContainer() (corev1.Container, error) +} + +// newSidecarProvider creates a provider-specific SidecarProvider for the given keyID, UDS endpoint, and plugin configuration. +func newSidecarProvider(keyID string, udsPath string, pluginConfig configv1.KMSPluginConfig) (sidecarProvider, error) { + switch pluginConfig.Type { + case configv1.VaultKMSProvider: + return newVaultSidecarProvider("vault-kms-plugin", keyID, udsPath, pluginConfig) + default: + return nil, fmt.Errorf("unsupported KMS plugin configuration") + } +} + +// AddKMSPluginSidecarToPodSpec discovers KMS plugins from the encryption-config secret and injects a sidecar container for each one into the pod spec. +// It is a no-op when the KMSEncryption feature gate is not enabled or the encryption-config secret does not exist. +// It uses an uncached client to avoid injecting sidecars based on a stale encryption configuration. +func AddKMSPluginSidecarToPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess) error { + if podSpec == nil { + return fmt.Errorf("pod spec cannot be nil") + } + + if containerName == "" { + return fmt.Errorf("container name cannot be empty") + } + + if !featureGateAccessor.AreInitialFeatureGatesObserved() { + return nil + } + + featureGates, err := featureGateAccessor.CurrentFeatureGates() + if err != nil { + return fmt.Errorf("failed to get feature gates: %w", err) + } + + if !featureGates.Enabled(features.FeatureGateKMSEncryption) { + return nil + } + + encryptionConfigurationSecret, err := secretClient.Secrets(encryptionConfigNamespace).Get(ctx, encryptionConfigSecretName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + klog.V(4).Infof("skipping KMS sidecar injection: %s/%s secret not found", encryptionConfigNamespace, encryptionConfigSecretName) + return nil + } + if err != nil { + return fmt.Errorf("failed to get %s/%s secret: %w", encryptionConfigNamespace, encryptionConfigSecretName, err) + } + + encryptionConfig, err := encryptiondata.FromSecret(encryptionConfigurationSecret) + if err != nil { + return fmt.Errorf("failed to extract encryption config from %s/%s secret: %w", encryptionConfigNamespace, encryptionConfigSecretName, err) + } + + kmsConfigurations, err := encryptiondata.ExtractUniqueAndSortedKMSConfigurations(encryptionConfig) + if err != nil { + return fmt.Errorf("failed to get KMS configurations: %w", err) + } + if len(kmsConfigurations) == 0 { + klog.V(4).Infof("skipping KMS sidecar injection: no KMS plugins found in EncryptionConfiguration") + return nil + } + + klog.V(4).Infof("injecting %d KMS sidecar(s)", len(kmsConfigurations)) + + for _, kmsConfiguration := range kmsConfigurations { + // ExtractUniqueAndSortedKMSConfigurations function rewrites the .Name field to include only the key ID + keyID := kmsConfiguration.Name + udsPath := kmsConfiguration.Endpoint + + pluginConfig, ok := encryptionConfig.KMSPlugins[keyID] + if !ok { + return fmt.Errorf("missing plugin config for keyID %s", keyID) + } + + sidecarProvider, err := newSidecarProvider(keyID, udsPath, pluginConfig) + if err != nil { + return fmt.Errorf("failed to create a sidecar provider for keyID %s: %w", keyID, err) + } + + if err := ensureSidecarContainer(podSpec, sidecarProvider); err != nil { + return err + } + + if err := ensureSocketVolumeMountInContainer(podSpec.InitContainers, sidecarProvider.Name()); err != nil { + return err + } + } + + if err := ensureSocketVolumeMountInContainer(podSpec.Containers, containerName); err != nil { + return err + } + + // The volume mount in the kube-apiserver and KMS plugin containers requires a volume in the podSpec + ensureSocketVolume(podSpec) + + return nil +} + +func ensureSidecarContainer(podSpec *corev1.PodSpec, provider sidecarProvider) error { + sidecar, err := provider.BuildSidecarContainer() + if err != nil { + return fmt.Errorf("failed to build sidecar container: %w", err) + } + + for i, container := range podSpec.InitContainers { + if container.Name == sidecar.Name { + podSpec.InitContainers[i] = sidecar + return nil + } + } + + podSpec.InitContainers = append(podSpec.InitContainers, sidecar) + return nil +} + +func ensureSocketVolumeMountInContainer(containers []corev1.Container, containerName string) error { + containerIndex := -1 + for i, container := range containers { + if container.Name == containerName { + containerIndex = i + break + } + } + + if containerIndex < 0 { + return fmt.Errorf("container %s not found", containerName) + } + + foundMount := false + container := &containers[containerIndex] + for _, m := range container.VolumeMounts { + if m.Name == "kms-plugin-socket" { + foundMount = true + break + } + } + if !foundMount { + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "kms-plugin-socket", + MountPath: "/var/run/kmsplugin", + }, + ) + } + return nil +} + +func ensureSocketVolume(podSpec *corev1.PodSpec) { + for _, volume := range podSpec.Volumes { + if volume.Name == "kms-plugin-socket" { + return + } + } + + podSpec.Volumes = append(podSpec.Volumes, + corev1.Volume{ + Name: "kms-plugin-socket", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go new file mode 100644 index 0000000000..937bdfd748 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go @@ -0,0 +1,77 @@ +package pluginlifecycle + +import ( + "fmt" + + configv1 "github.com/openshift/api/config/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +// newVaultSidecarProvider creates a Vault sidecar provider from the given KMS plugin configuration. +func newVaultSidecarProvider(name, keyID, udsPath string, pluginConfig configv1.KMSPluginConfig) (*vault, error) { + return &vault{ + name: name, + keyID: keyID, + udsPath: udsPath, + config: &pluginConfig.Vault, + }, nil +} + +// vault implements SidecarProvider for HashiCorp Vault KMS. +type vault struct { + name string + keyID string + udsPath string + config *configv1.VaultKMSPluginConfig +} + +// Name returns the sidecar name appended by the key id. +func (v *vault) Name() string { + return fmt.Sprintf("%s-%s", v.name, v.keyID) +} + +// BuildSidecarContainer returns a container spec for the Vault KMS plugin sidecar +// configured with the Vault address, namespace, transit mount, and transit key. +func (v *vault) BuildSidecarContainer() (corev1.Container, error) { + // Required API fields: always set. + args := []string{ + fmt.Sprintf("-listen-address=%s", v.udsPath), + fmt.Sprintf("-vault-address=%s", v.config.VaultAddress), + fmt.Sprintf("-transit-key=%s", v.config.TransitKey), + // TODO(bertinatto): dummy value for the Vault mock plugin; will come from the encryption-config secret. + fmt.Sprintf("-approle-role-id=dummy-role-id-%s", v.keyID), + // TODO(bertinatto): placeholder path for the Vault mock plugin; will differ per operator (KASO vs. aggregated apiserver operators). + fmt.Sprintf("-approle-secret-id-path=/var/run/secrets/vault-kms/secret-id-%s", v.keyID), + } + + // Optional fields: only pass non-empty values. + if v.config.VaultNamespace != "" { + args = append(args, fmt.Sprintf("-vault-namespace=%s", v.config.VaultNamespace)) + } + + if v.config.TransitMount != "" { + args = append(args, fmt.Sprintf("-transit-mount=%s", v.config.TransitMount)) + } + + return corev1.Container{ + Name: v.Name(), + Image: v.config.KMSPluginImage, + Args: args, + ImagePullPolicy: corev1.PullIfNotPresent, + // We place the container in InitContainers with RestartPolicyAlways so the kubelet starts it before + // regular containers and keeps it running for the pod's lifetime. + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + // TODO(bertinatto): the plugin sidecar needs to be measure under heavy load to figure out good defaults. + // For now follow what most sidecars in the kube-apiserver pod do. xref: + // https://github.com/openshift/cluster-kube-apiserver-operator/commit/e15a19cd2474c8b60ce17ac16dd8f422c729847a + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("50Mi"), + corev1.ResourceCPU: resource.MustParse("5m"), + }, + }, + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d836054a00..172bd78f11 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -446,6 +446,7 @@ github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata github.com/openshift/library-go/pkg/operator/encryption/kms +github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets github.com/openshift/library-go/pkg/operator/encryption/state From 204f7371a5608c96ca9864bbd61f1b0d64606d33 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Tue, 26 May 2026 13:51:12 +0530 Subject: [PATCH 3/4] new test --- test/e2e-encryption-kms/encryption_kms.go | 104 ++++++++++++++++++---- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/test/e2e-encryption-kms/encryption_kms.go b/test/e2e-encryption-kms/encryption_kms.go index 34569175c1..f58e7a99ce 100644 --- a/test/e2e-encryption-kms/encryption_kms.go +++ b/test/e2e-encryption-kms/encryption_kms.go @@ -5,23 +5,31 @@ import ( "fmt" "math/rand/v2" "testing" + "time" g "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" - configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption" library "github.com/openshift/library-go/test/library/encryption" librarykms "github.com/openshift/library-go/test/library/encryption/kms" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { - g.It("TestKMSEncryptionOnOff [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func() { - testKMSEncryptionOnOff(g.GinkgoTB()) + g.It("TestKMSEncryptionOnOff [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) { + testKMSEncryptionOnOff(ctx, g.GinkgoTB()) }) - g.It("TestKMSEncryptionProvidersMigration [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func() { - testKMSEncryptionProvidersMigration(g.GinkgoTB()) + g.It("TestKMSEncryptionProvidersMigration [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) { + testKMSEncryptionProvidersMigration(ctx, g.GinkgoTB()) + }) + + g.It("TestKMSEncryptionInvalidImageRecovery [OCPFeatureGate:KMSEncryption][Serial][Timeout:120m]", func(ctx context.Context) { + testKMSEncryptionInvalidImageRecovery(ctx, g.GinkgoTB()) }) }) @@ -37,12 +45,12 @@ var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { // 8. Verifies secret is encrypted again // 9. Disables encryption (Identity) again // 10. Verifies secret is NOT encrypted again -func testKMSEncryptionOnOff(t testing.TB) { +func testKMSEncryptionOnOff(ctx context.Context, t testing.TB) { // Deploy the mock KMS plugin for testing. // NOTE: This manual deployment is only required for KMS v1. In the future, // the platform will manage the KMS plugins, and this code will no longer be needed. - librarykms.DeployUpstreamMockKMSPlugin(context.Background(), t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) - library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{ + librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) + library.TestEncryptionTurnOnAndOff(ctx, t, library.OnOffScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, @@ -57,10 +65,7 @@ func testKMSEncryptionOnOff(t testing.TB) { AssertResourceNotEncryptedFunc: operatorencryption.AssertSecretOfLifeNotEncrypted, ResourceFunc: operatorencryption.SecretOfLife, ResourceName: "SecretOfLife", - EncryptionProvider: library.EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{ - Type: configv1.EncryptionTypeKMS, - KMS: librarykms.DefaultFakeKMSPluginConfig, - }}, + EncryptionProvider: librarykms.DefaultFakeVaultEncryptionProvider, }) } @@ -72,9 +77,9 @@ func testKMSEncryptionOnOff(t testing.TB) { // 4. Shuffles the selected AES provider with KMS to create a randomized migration order // 5. Migrates between the providers in the shuffled order // 6. Verifies secret is correctly encrypted after each migration -func testKMSEncryptionProvidersMigration(t testing.TB) { - librarykms.DeployUpstreamMockKMSPlugin(context.Background(), t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) - library.TestEncryptionProvidersMigration(t, library.ProvidersMigrationScenario{ +func testKMSEncryptionProvidersMigration(ctx context.Context, t testing.TB) { + librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) + library.TestEncryptionProvidersMigration(ctx, t, library.ProvidersMigrationScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, @@ -90,8 +95,75 @@ func testKMSEncryptionProvidersMigration(t testing.TB) { ResourceFunc: operatorencryption.SecretOfLife, ResourceName: "SecretOfLife", EncryptionProviders: library.ShuffleEncryptionProviders([]library.EncryptionProvider{ - {APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeKMS, KMS: librarykms.DefaultFakeKMSPluginConfig}}, + librarykms.DefaultFakeVaultEncryptionProvider, library.SupportedStaticEncryptionProviders[rand.IntN(len(library.SupportedStaticEncryptionProviders))], }), }) } + +// testKMSEncryptionInvalidImageRecovery tests that an invalid KMS plugin image +// causes degradation and that fixing the image restores the cluster. +func testKMSEncryptionInvalidImageRecovery(ctx context.Context, t testing.TB) { + librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) + + invalidProvider := librarykms.DefaultFakeVaultEncryptionProvider + invalidProvider.KMS.Vault.KMSPluginImage = "quay.io/openshift/invalid-kms-image:does-not-exist" + + library.TestEncryptionInvalidImageRecovery(ctx, t, library.InvalidImageRecoveryScenario{ + BasicScenario: library.BasicScenario{ + Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), + EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + OperatorNamespace: operatorclient.OperatorNamespace, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertSecretsAndConfigMaps, + }, + InvalidImageProvider: invalidProvider, + ValidImageProvider: librarykms.DefaultFakeVaultEncryptionProvider, + WaitForDegraded: func(ctx context.Context, t testing.TB) { + t.Helper() + operatorClient := operatorencryption.GetOperator(t) + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { + operator, err := operatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false, nil + } + for _, cond := range operator.Status.Conditions { + if cond.Type == "Degraded" && cond.Status == operatorv1.ConditionTrue { + t.Logf("Operator is degraded: %s", cond.Message) + return true, nil + } + } + return false, nil + }) + require.NoError(t, err, "timed out waiting for operator to become degraded") + }, + WaitForRecovery: func(ctx context.Context, t testing.TB) { + t.Helper() + operatorClient := operatorencryption.GetOperator(t) + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 20*time.Minute, true, func(ctx context.Context) (bool, error) { + operator, err := operatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false, nil + } + degraded := false + progressing := false + for _, cond := range operator.Status.Conditions { + if cond.Type == "Degraded" && cond.Status == operatorv1.ConditionTrue { + degraded = true + } + if cond.Type == "Progressing" && cond.Status == operatorv1.ConditionTrue { + progressing = true + } + } + if !degraded && !progressing { + t.Log("Operator recovered: not degraded and not progressing") + return true, nil + } + return false, nil + }) + require.NoError(t, err, "timed out waiting for operator to recover") + }, + }) +} From 98a3204be8fb3d4c991d32a66ed65685eb101124 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Tue, 26 May 2026 13:51:40 +0530 Subject: [PATCH 4/4] go mod vendor --- go.mod | 2 + go.sum | 4 +- test/e2e-encryption-kms/encryption_kms.go | 68 ++------ .../encryption_perf_test.go | 2 +- .../encryption_rotation_test.go | 2 +- test/e2e-encryption/encryption_test.go | 6 +- test/e2e/encryption.go | 9 +- test/e2e/encryption_test.go | 2 +- .../encryption/controllers/key_controller.go | 152 +++++++++++++++++- .../controllers/state_controller.go | 86 ++++++++++ .../encryption/encryptiondata/config.go | 71 +++++++- .../encryption/encryptiondata/secret.go | 88 +++++++++- .../pkg/operator/encryption/kms/helpers.go | 92 ----------- .../operator/encryption/secrets/secrets.go | 26 ++- .../pkg/operator/encryption/secrets/types.go | 9 +- .../pkg/operator/encryption/state/types.go | 63 ++++++++ .../test/library/encryption/helpers.go | 34 +++- .../test/library/encryption/kms/vault.go | 67 ++++++-- .../test/library/encryption/perf_scenarios.go | 9 +- .../test/library/encryption/scenarios.go | 123 ++++++++++---- vendor/modules.txt | 4 +- 21 files changed, 689 insertions(+), 230 deletions(-) delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/helpers.go diff --git a/go.mod b/go.mod index 6eabf54ecb..68bd7c2a81 100644 --- a/go.mod +++ b/go.mod @@ -136,3 +136,5 @@ require ( ) replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 + +replace github.com/openshift/library-go => github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86 diff --git a/go.sum b/go.sum index d013390c0c..3eb42d76f2 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86 h1:A83Pqt6r8r2v/1I3mARfUS0aD7P3BNEyFAUy1vtcrRg= +github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -165,8 +167,6 @@ github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+S github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a h1:EKx2XhOKehd1C5ptY7IrLl4WV35E8kP0pRPnG5BUZXk= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a/go.mod h1:V933kvY/cb/Un7UCEOhXHUySNX327u7Epe8g9KNqg2Q= -github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9 h1:1ubwPydT+ABjfvmeiv4hoJQ0gIDCyMq/U5UyHrrpefA= -github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 h1:PMTgifBcBRLJJiM+LgSzPDTk9/Rx4qS09OUrfpY6GBQ= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= diff --git a/test/e2e-encryption-kms/encryption_kms.go b/test/e2e-encryption-kms/encryption_kms.go index f58e7a99ce..121acc671f 100644 --- a/test/e2e-encryption-kms/encryption_kms.go +++ b/test/e2e-encryption-kms/encryption_kms.go @@ -8,15 +8,12 @@ import ( "time" g "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/require" - operatorv1 "github.com/openshift/api/operator/v1" "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption" + libraryapiserver "github.com/openshift/library-go/test/library/apiserver" library "github.com/openshift/library-go/test/library/encryption" librarykms "github.com/openshift/library-go/test/library/encryption/kms" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" ) var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { @@ -65,7 +62,7 @@ func testKMSEncryptionOnOff(ctx context.Context, t testing.TB) { AssertResourceNotEncryptedFunc: operatorencryption.AssertSecretOfLifeNotEncrypted, ResourceFunc: operatorencryption.SecretOfLife, ResourceName: "SecretOfLife", - EncryptionProvider: librarykms.DefaultFakeVaultEncryptionProvider, + EncryptionProvider: library.EncryptionProvider{APIServerEncryption: librarykms.DefaultFakeKMSPluginConfig}, }) } @@ -95,7 +92,7 @@ func testKMSEncryptionProvidersMigration(ctx context.Context, t testing.TB) { ResourceFunc: operatorencryption.SecretOfLife, ResourceName: "SecretOfLife", EncryptionProviders: library.ShuffleEncryptionProviders([]library.EncryptionProvider{ - librarykms.DefaultFakeVaultEncryptionProvider, + {APIServerEncryption: librarykms.DefaultFakeKMSPluginConfig}, library.SupportedStaticEncryptionProviders[rand.IntN(len(library.SupportedStaticEncryptionProviders))], }), }) @@ -105,10 +102,6 @@ func testKMSEncryptionProvidersMigration(ctx context.Context, t testing.TB) { // causes degradation and that fixing the image restores the cluster. func testKMSEncryptionInvalidImageRecovery(ctx context.Context, t testing.TB) { librarykms.DeployUpstreamMockKMSPlugin(ctx, t, library.GetClients(t).Kube, librarykms.WellKnownUpstreamMockKMSPluginNamespace, librarykms.WellKnownUpstreamMockKMSPluginImage, librarykms.DefaultKMSPluginCount) - - invalidProvider := librarykms.DefaultFakeVaultEncryptionProvider - invalidProvider.KMS.Vault.KMSPluginImage = "quay.io/openshift/invalid-kms-image:does-not-exist" - library.TestEncryptionInvalidImageRecovery(ctx, t, library.InvalidImageRecoveryScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, @@ -119,51 +112,20 @@ func testKMSEncryptionInvalidImageRecovery(ctx context.Context, t testing.TB) { TargetGRs: operatorencryption.DefaultTargetGRs, AssertFunc: operatorencryption.AssertSecretsAndConfigMaps, }, - InvalidImageProvider: invalidProvider, - ValidImageProvider: librarykms.DefaultFakeVaultEncryptionProvider, + InvalidImageProvider: librarykms.InvalidImageVaultEncryptionProvider, + ValidImageProvider: librarykms.DefaultVaultEncryptionProvider, WaitForDegraded: func(ctx context.Context, t testing.TB) { t.Helper() - operatorClient := operatorencryption.GetOperator(t) - err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { - operator, err := operatorClient.Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return false, nil - } - for _, cond := range operator.Status.Conditions { - if cond.Type == "Degraded" && cond.Status == operatorv1.ConditionTrue { - t.Logf("Operator is degraded: %s", cond.Message) - return true, nil - } - } - return false, nil - }) - require.NoError(t, err, "timed out waiting for operator to become degraded") - }, - WaitForRecovery: func(ctx context.Context, t testing.TB) { - t.Helper() - operatorClient := operatorencryption.GetOperator(t) - err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 20*time.Minute, true, func(ctx context.Context) (bool, error) { - operator, err := operatorClient.Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return false, nil - } - degraded := false - progressing := false - for _, cond := range operator.Status.Conditions { - if cond.Type == "Degraded" && cond.Status == operatorv1.ConditionTrue { - degraded = true - } - if cond.Type == "Progressing" && cond.Status == operatorv1.ConditionTrue { - progressing = true - } - } - if !degraded && !progressing { - t.Log("Operator recovered: not degraded and not progressing") - return true, nil - } - return false, nil - }) - require.NoError(t, err, "timed out waiting for operator to recover") + cs := library.GetClients(t) + library.WaitForPodImagePullBackOff(ctx, t, cs.Kube, operatorclient.TargetNamespace, "app=openshift-kube-apiserver", 15*time.Minute) }, }) + + // wait for all kube-apiserver pods to stabilize on the same revision after recovery + cs := library.GetClients(t) + podClient := cs.Kube.CoreV1().Pods(operatorclient.TargetNamespace) + err := libraryapiserver.WaitForAPIServerToStabilizeOnTheSameRevision(t, podClient) + if err != nil { + t.Fatalf("apiserver pods did not stabilize after recovery: %v", err) + } } diff --git a/test/e2e-encryption-perf/encryption_perf_test.go b/test/e2e-encryption-perf/encryption_perf_test.go index 7540e280ee..6d496e6f09 100644 --- a/test/e2e-encryption-perf/encryption_perf_test.go +++ b/test/e2e-encryption-perf/encryption_perf_test.go @@ -31,7 +31,7 @@ var provider = flag.String("provider", "aescbc", "encryption provider used by th func TestPerfEncryption(tt *testing.T) { operatorClient := operatorencryption.GetOperator(tt) - library.TestPerfEncryption(tt, library.PerfScenario{ + library.TestPerfEncryption(tt.Context(), tt, library.PerfScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, diff --git a/test/e2e-encryption-rotation/encryption_rotation_test.go b/test/e2e-encryption-rotation/encryption_rotation_test.go index bb9e1bc065..496412df30 100644 --- a/test/e2e-encryption-rotation/encryption_rotation_test.go +++ b/test/e2e-encryption-rotation/encryption_rotation_test.go @@ -19,7 +19,7 @@ var provider = flag.String("provider", "aescbc", "encryption provider used by th // rotation by setting the "encyrption.Reason" in the operator's configuration // file func TestEncryptionRotation(t *testing.T) { - library.TestEncryptionRotation(t, library.RotationScenario{ + library.TestEncryptionRotation(t.Context(), t, library.RotationScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 4e49807269..60d933b299 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -14,7 +14,7 @@ import ( var provider = flag.String("provider", "aescbc", "encryption provider used by the tests") func TestEncryptionTypeIdentity(t *testing.T) { - library.TestEncryptionTypeIdentity(t, library.BasicScenario{ + library.TestEncryptionTypeIdentity(t.Context(), t, library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), @@ -26,7 +26,7 @@ func TestEncryptionTypeIdentity(t *testing.T) { } func TestEncryptionTypeUnset(t *testing.T) { - library.TestEncryptionTypeUnset(t, library.BasicScenario{ + library.TestEncryptionTypeUnset(t.Context(), t, library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), @@ -38,7 +38,7 @@ func TestEncryptionTypeUnset(t *testing.T) { } func TestEncryptionTurnOnAndOff(t *testing.T) { - library.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{ + library.TestEncryptionTurnOnAndOff(t.Context(), t, library.OnOffScenario{ BasicScenario: library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, diff --git a/test/e2e/encryption.go b/test/e2e/encryption.go index 12b7267144..ca16a65a1e 100644 --- a/test/e2e/encryption.go +++ b/test/e2e/encryption.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "fmt" "testing" @@ -12,13 +13,13 @@ import ( ) var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { - g.It("[Operator][Serial][Timeout:40m] TestEncryptionTypeAESCBC", func() { - testEncryptionTypeAESCBC(g.GinkgoTB()) + g.It("[Operator][Serial][Timeout:40m] TestEncryptionTypeAESCBC", func(ctx context.Context) { + testEncryptionTypeAESCBC(ctx, g.GinkgoTB()) }) }) -func testEncryptionTypeAESCBC(t testing.TB) { - library.TestEncryptionTypeAESCBC(t, library.BasicScenario{ +func testEncryptionTypeAESCBC(ctx context.Context, t testing.TB) { + library.TestEncryptionTypeAESCBC(ctx, t, library.BasicScenario{ Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), diff --git a/test/e2e/encryption_test.go b/test/e2e/encryption_test.go index 5c41b02104..3a95aa4ad7 100644 --- a/test/e2e/encryption_test.go +++ b/test/e2e/encryption_test.go @@ -10,5 +10,5 @@ import ( // This situation is temporary until we test the new e2e-gcp-operator-serial-ote job. // Eventually all tests will be run only as part of the OTE framework. func TestEncryptionTypeAESCBC(t *testing.T) { - testEncryptionTypeAESCBC(t) + testEncryptionTypeAESCBC(t.Context(), t) } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go index 3c930469d5..0a2032eedf 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go @@ -11,6 +11,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -28,6 +29,7 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/encryption/crypto" + "github.com/openshift/library-go/pkg/operator/encryption/encoding" "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/state" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" @@ -42,6 +44,7 @@ const ( encryptionSecretMigrationInterval = time.Hour * 24 * 7 // one week kmsEndpointFormat = "unix:///var/run/kmsplugin/kms-%d.sock" defaultKMSTimeout = 10 * time.Second + openshiftConfigNS = "openshift-config" ) // keyController creates new keys if necessary. It @@ -117,6 +120,7 @@ func NewKeyController( apiServerInformer.Informer(), operatorClient.Informer(), kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), + // TODO: add informer for openshift-config namespace to watch referenced Secrets for KMS plugin secret data changes deployer, ).ToController( c.controllerInstanceName, @@ -169,6 +173,16 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact return err } + // Apply in-place KMS plugin config updates (e.g. image, TLS) to the latest key + // secret regardless of convergence. This unblocks stuck revisions and propagates + // operational fixes like CVE image updates. Changes to migration-triggering fields + // (transit key, vault address) are skipped via kmsMigrationRequired. + if currentMode == state.KMS { + if err := c.maybeUpdateKMSPluginConfigInPlace(ctx, syncContext, apiEncryptionConfiguration); err != nil { + return err + } + } + currentConfig, desiredEncryptionState, secrets, isProgressingReason, err := statemachine.GetEncryptionConfigAndState(ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, encryptedGRs) if err != nil { return err @@ -223,7 +237,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact sort.Sort(sort.StringSlice(reasons)) internalReason := strings.Join(reasons, ", ") - keySecret, err := c.generateKeySecret(newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) + keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) if err != nil { return fmt.Errorf("failed to create key: %v", err) } @@ -260,7 +274,104 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c return nil // we made this key earlier } -func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { +// maybeUpdateKMSPluginConfigInPlace updates the latest key secret's KMS plugin +// config when only in-place-safe fields changed (image, TLS, authentication). +func (c *keyController) maybeUpdateKMSPluginConfigInPlace(ctx context.Context, syncContext factory.SyncContext, apiServerEncryption configv1.APIServerEncryption) error { + keySecrets, err := secrets.ListKeySecrets(ctx, c.secretClient, c.encryptionSecretSelector) + if err != nil { + return err + } + if len(keySecrets) == 0 { + return nil + } + // Sort by key ID descending so [0] is the newest. + // Parse the newest secret directly — fail fast if it is malformed + // instead of silently falling back to an older key. + sort.Slice(keySecrets, func(i, j int) bool { + iKeyID, _ := state.NameToKeyID(keySecrets[i].Name) + jKeyID, _ := state.NameToKeyID(keySecrets[j].Name) + return iKeyID > jKeyID + }) + // We only focus on the latest backed key. + latest, err := secrets.ToKeyState(keySecrets[0]) + if err != nil { + return fmt.Errorf("latest key secret %s is invalid: %w", keySecrets[0].Name, err) + } + + // Any mode mismatch (e.g. KMS <-> AESCBC) requires a migration, not an in-place + // update. The normal needsNewKey path handles this after convergence. + if latest.Mode != state.KMS { + return nil + } + // This should never happen under normal operation because ToKeyState enforces + // that KMS mode keys have a plugin config. This can only occur if someone + // manually edited the key secret and removed the kms-plugin-config data field. + // To mitigate, re-add the removed kms-plugin-config data to the key secret. + if !latest.HasKMSPlugin() { + return fmt.Errorf("latest KMS key %s is missing plugin config", latest.Key.Name) + } + // Skip when the plugin config is already up-to-date or when + if equality.Semantic.DeepEqual(latest.KMS.Plugin, apiServerEncryption.KMS) { + return nil + } + + migrationRequired, err := kmsMigrationRequired(latest.KMS.Plugin, apiServerEncryption.KMS) + if err != nil { + return err + } + // migration-triggering fields changed (needs a new key, not an in-place update). + if migrationRequired { + return nil + } + + keySecret, err := secrets.FromKeyState(c.instanceName, latest) + if err != nil { + return err + } + s, err := c.secretClient.Secrets(keySecret.Namespace).Get(ctx, keySecret.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get key secret %s/%s: %v", keySecret.Namespace, keySecret.Name, err) + } + pluginData, err := encoding.EncodeKMSPluginConfig(apiServerEncryption.KMS) + if err != nil { + return fmt.Errorf("failed to encode KMS plugin config: %v", err) + } + s.Data["encryption.apiserver.operator.openshift.io-kms-plugin-config"] = pluginData + _, updateErr := c.secretClient.Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + if errors.IsConflict(updateErr) { + return nil + } + if updateErr == nil { + syncContext.Recorder().Eventf("EncryptionKeyKMSPluginConfigUpdated", "Updated KMS plugin config on key secret %q in-place", s.Name) + } + return updateErr +} + +// kmsMigrationRequired reports whether the KMS config change between latest +// (stored in the key secret) and current (from the APIServer CR) involves +// migration-triggering fields that require a new encryption key. +// Returns false when only in-place-safe fields differ (image, TLS, authentication) +// or when configs are identical. +func kmsMigrationRequired(latest, current configv1.KMSPluginConfig) (bool, error) { + if equality.Semantic.DeepEqual(latest, current) { + return false, nil + } + if latest.Type != configv1.VaultKMSProvider || current.Type != configv1.VaultKMSProvider { + return false, fmt.Errorf("KMS plugin config has an invalid type: %q", latest.Type) + } + if latest.Type != current.Type { + return true, nil + } + if latest.Vault.VaultAddress != current.Vault.VaultAddress || + latest.Vault.VaultNamespace != current.Vault.VaultNamespace || + latest.Vault.TransitMount != current.Vault.TransitMount || + latest.Vault.TransitKey != current.Vault.TransitKey { + return true, nil + } + return false, nil +} + +func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { bs := crypto.ModeToNewKeyFunc[currentMode]() ks := state.KeyState{ Key: apiserverv1.Key{ @@ -281,6 +392,24 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, }, Plugin: apiServerEncryption.KMS, } + + if secretName, expectedKeys, err := referencedSecretName(apiServerEncryption.KMS); err != nil { + return nil, err + } else if len(secretName) > 0 { + refSecret, err := c.secretClient.Secrets(openshiftConfigNS).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s in %s: %w", secretName, openshiftConfigNS, err) + } + for _, key := range expectedKeys { + v, ok := refSecret.Data[key] + if !ok { + return nil, fmt.Errorf("secret %s in %s is missing required key %q", secretName, openshiftConfigNS, key) + } + if err := ks.KMS.PluginSecretData.Set(secretName, key, v); err != nil { + return nil, err + } + } + } } return secrets.FromKeyState(c.instanceName, ks) } @@ -383,6 +512,25 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return latestKeyID, "rotation-interval-has-passed", time.Since(latestKey.Migrated.Timestamp) > encryptionSecretMigrationInterval } +// referencedSecretName returns the name of the secret referenced by the KMS plugin +// config and the specific data keys to carry from that secret. Only the listed keys +// are copied into the Key Secret; any other data in the referenced secret is ignored. +func referencedSecretName(plugin configv1.KMSPluginConfig) (string, []string, error) { + switch plugin.Type { + case configv1.VaultKMSProvider: + switch plugin.Vault.Authentication.Type { + case configv1.VaultAuthenticationTypeAppRole: + // The Vault AppRole secret must contain "role-id" and "secret-id" keys. + // These are the only keys carried into the encryption key secret. + return plugin.Vault.Authentication.AppRole.Secret.Name, []string{"role-id", "secret-id"}, nil + default: + return "", nil, fmt.Errorf("unsupported Vault authentication type %q", plugin.Vault.Authentication.Type) + } + default: + return "", nil, fmt.Errorf("unsupported KMS provider type %q", plugin.Type) + } +} + // TODO make this un-settable once set // ex: we could require the tech preview no upgrade flag to be set before we will honor this field type unsupportedEncryptionConfig struct { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/state_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/state_controller.go index a0b7f1f677..4ef4c5982e 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/state_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/state_controller.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" @@ -16,6 +18,7 @@ import ( configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/encryption/encryptiondata" + "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/state" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" "github.com/openshift/library-go/pkg/operator/events" @@ -132,6 +135,11 @@ func (c *stateController) generateAndApplyCurrentEncryptionConfigSecret(ctx cont return err } if len(transitioningReason) > 0 { + // Even when not converged, propagate in-place KMS plugin config changes + // so the revision controller can create a new revision with corrected sidecar config. + if err := c.maybeUpdateKMSDataInEncryptionConfigSecret(ctx, recorder); err != nil { + return err + } queue.AddAfter(stateWorkKey, 2*time.Minute) return nil } @@ -163,6 +171,84 @@ func (c *stateController) generateAndApplyCurrentEncryptionConfigSecret(ctx cont return nil } +// maybeUpdateKMSDataInEncryptionConfigSecret propagates in-place KMS plugin +// config changes to the encryption-config secret during non-convergence. It enriches +// the existing state with current key secrets (picking up updated plugin configs from +// key_controller), guards that the EncryptionConfiguration is unchanged (no key +// promotion or structural changes), and applies only kms-plugin-config updates. +func (c *stateController) maybeUpdateKMSDataInEncryptionConfigSecret(ctx context.Context, recorder events.Recorder) error { + keySecrets, err := secrets.ListKeySecrets(ctx, c.secretClient, c.encryptionSecretSelector) + if err != nil { + return err + } + if len(keySecrets) == 0 { + return nil + } + name := fmt.Sprintf("%s-%s", encryptiondata.EncryptionConfSecretName, c.instanceName) + namespace := "openshift-config-managed" + // Read the unrevisioned encryption-config from openshift-config-managed (not the + // revisioned copy in the target namespace) because during non-convergence the deployer + // cannot return a single converged revision. Updating this source secret triggers + // resource-sync → revision → rollout to unblock the stuck revision. + existingSecret, err := c.secretClient.Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + + existingConfig, err := encryptiondata.FromSecret(existingSecret) + if err != nil { + return err + } + if existingConfig == nil || len(existingConfig.KMSPlugins) == 0 { + return nil + } + + // Round-trip through ToEncryptionState → FromEncryptionState to pick up + // updated KMS plugin configs from key secrets while preserving the existing + // EncryptionConfiguration structure. ToEncryptionState enriches each key in + // the config with its backed secret data (including any in-place plugin config + // updates), and FromEncryptionState rebuilds the Config from the enriched state. + enrichedState, _ := encryptiondata.ToEncryptionState(existingConfig, keySecrets) + if enrichedState == nil { + return nil + } + + rebuiltConfig, err := encryptiondata.FromEncryptionState(enrichedState) + if err != nil { + return err + } + + // Only proceed if the provider list, key ordering, and write key designation + // are unchanged. Structural changes require convergence to avoid a server + // encrypting with a key another server hasn't observed. + if !equality.Semantic.DeepEqual(existingConfig.Encryption.Resources, rebuiltConfig.Encryption.Resources) { + return nil + } + + // Error if the plugin key set changed unexpectedly — this indicates + // a bug in the enrichment round-trip or corrupted state. + if len(existingConfig.KMSPlugins) != len(rebuiltConfig.KMSPlugins) { + return fmt.Errorf("KMS plugin key set size changed unexpectedly: existing=%d, rebuilt=%d", len(existingConfig.KMSPlugins), len(rebuiltConfig.KMSPlugins)) + } + for keyID := range existingConfig.KMSPlugins { + if _, ok := rebuiltConfig.KMSPlugins[keyID]; !ok { + return fmt.Errorf("KMS plugin config for keyID %s disappeared after enrichment round-trip", keyID) + } + } + + changed, err := c.applyEncryptionConfigSecret(ctx, rebuiltConfig, recorder) + if err != nil { + return err + } + if changed { + recorder.Eventf("EncryptionKMSPluginConfigPropagated", "Updated KMS plugin config in encryption-config secret %s/%s during non-convergence", namespace, name) + } + return nil +} + func (c *stateController) applyEncryptionConfigSecret(ctx context.Context, secretData *encryptiondata.Config, recorder events.Recorder) (bool, error) { s, err := encryptiondata.ToSecret("openshift-config-managed", fmt.Sprintf("%s-%s", encryptiondata.EncryptionConfSecretName, c.instanceName), secretData) if err != nil { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go index 40a855a81f..c4b7706af7 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go @@ -30,6 +30,48 @@ type Config struct { // KMSPlugins maps keyID to plugin-specific configuration, // carried from Key Secrets into the encryption-config Secret. KMSPlugins map[string]configv1.KMSPluginConfig + // KMSPluginsSecretData maps keyID to secret data carried from + // Key Secrets into the encryption-config Secret. + // Structure: keyID → secretName → dataKey → value. + KMSPluginsSecretData KMSPluginsSecretData +} + +// KMSPluginsSecretData maps keyID to secret data carried from Key Secrets into +// the encryption-config Secret. Structure: keyID → secretName → dataKey → value. +type KMSPluginsSecretData struct { + ByKeyID map[string]state.KMSSecretData +} + +// SetFromRawKey stores a value for the given keyID, splitting rawKey +// on "_" into secretName and dataKey. +func (d *KMSPluginsSecretData) SetFromRawKey(keyID, rawKey string, value []byte) error { + if len(keyID) == 0 { + return fmt.Errorf("keyID must not be empty") + } + if d.ByKeyID == nil { + d.ByKeyID = map[string]state.KMSSecretData{} + } + sd := d.ByKeyID[keyID] + if err := sd.SetFromRawKey(rawKey, value); err != nil { + return err + } + d.ByKeyID[keyID] = sd + return nil +} + +// FlatEntriesByKeyID returns the stored data as a map of keyID to flat entries, +// where each flat entry is keyed by "secretName_dataKey". +func (d *KMSPluginsSecretData) FlatEntriesByKeyID() map[string]map[string][]byte { + if d.ByKeyID == nil { + return nil + } + result := map[string]map[string][]byte{} + for keyID, sd := range d.ByKeyID { + if flat := sd.FlatEntries(); flat != nil { + result[keyID] = flat + } + } + return result } func (c *Config) HasEncryptionConfiguration() bool { @@ -40,6 +82,7 @@ func (c *Config) HasEncryptionConfiguration() bool { func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) { resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState)) var kmsPlugins map[string]configv1.KMSPluginConfig + var kmsPluginsSecretData KMSPluginsSecretData for gr, grKeys := range encryptionState { resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{ @@ -47,11 +90,11 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes Providers: stateToProviders(gr.Resource, grKeys), }) - // Collect KMS plugin configs from read keys (which already include the write key). - // We iterate over encryptionState which is keyed by GroupResource, so the same - // keyID is seen once per resource (e.g. key "1" for secrets and key "1" for configmaps). - // Since all resources share the same Key Secret, the plugin config is identical - // across duplicates and we only need to keep the first occurrence. + // Collect KMS plugin configs and secret data from read keys (which already + // include the write key). We iterate over encryptionState which is keyed by + // GroupResource, so the same keyID is seen once per resource. Since all + // resources share the same Key Secret, the plugin config and secret data are + // identical across duplicates and we only need to keep the first occurrence. for _, key := range grKeys.ReadKeys { if key.HasKMSPlugin() { if kmsPlugins == nil { @@ -67,6 +110,19 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes kmsPlugins[key.Key.Name] = key.KMS.Plugin } } + if key.HasKMSSecretData() { + if existing, exists := kmsPluginsSecretData.ByKeyID[key.Key.Name]; exists { + if !equality.Semantic.DeepEqual(existing, key.KMS.PluginSecretData) { + return nil, fmt.Errorf("KMS secret data mismatch for keyID %s: secret data from different resources must be identical", key.Key.Name) + } + } else { + for rawKey, value := range key.KMS.PluginSecretData.FlatEntries() { + if err := kmsPluginsSecretData.SetFromRawKey(key.Key.Name, rawKey, value); err != nil { + return nil, fmt.Errorf("failed to copy secret data for keyID %s: %w", key.Key.Name, err) + } + } + } + } } } @@ -76,8 +132,9 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes }) return &Config{ - Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, - KMSPlugins: kmsPlugins, + Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, + KMSPlugins: kmsPlugins, + KMSPluginsSecretData: kmsPluginsSecretData, }, nil } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/secret.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/secret.go index 0df884c62c..b2d025e19f 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/secret.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/secret.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strconv" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -13,15 +14,42 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/library-go/pkg/operator/encryption/encoding" - "github.com/openshift/library-go/pkg/operator/encryption/kms" "github.com/openshift/library-go/pkg/operator/encryption/state" ) -// EncryptionConfSecretName is the name of the final encryption config secret that is revisioned per apiserver rollout. -const EncryptionConfSecretName = "encryption-config" +const pluginConfigDataKeyPrefix = "kms-plugin-config-" -// EncryptionConfSecretKey is the map data key used to store the raw bytes of the final encryption config. -const EncryptionConfSecretKey = "encryption-config" +// toPluginConfigSecretDataKeyFor constructs the data key for storing a KMS plugin config in the encryption-config Secret. +// The keyID must be a valid non-negative integer string. +func toPluginConfigSecretDataKeyFor(keyID string) (string, error) { + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return pluginConfigDataKeyPrefix + keyID, nil +} + +// keyIDFromPluginConfigSecretDataKey extracts the keyID from a kms-plugin-config data key. +// Returns the keyID and true if the key matches the "kms-plugin-config-" pattern. +func keyIDFromPluginConfigSecretDataKey(dataKey string) (string, bool, error) { + keyID, found := strings.CutPrefix(dataKey, pluginConfigDataKeyPrefix) + if !found || len(keyID) == 0 { + return "", false, nil + } + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return keyID, true, nil +} + +const ( + // EncryptionConfSecretName is the name of the final encryption config secret that is revisioned per apiserver rollout. + EncryptionConfSecretName = "encryption-config" + // EncryptionConfSecretKey is the map data key used to store the raw bytes of the final encryption config. + EncryptionConfSecretKey = "encryption-config" + // encryptionConfigSecretDataPrefix is the data key prefix for KMS plugin secret + // data entries in the encryption-config Secret. Full key: "kms-plugin-secret-{secretName}_{dataKey}-{keyID}". + encryptionConfigSecretDataPrefix = "kms-plugin-secret-" +) func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { data, ok := encryptionConfigSecret.Data[EncryptionConfSecretKey] @@ -36,7 +64,7 @@ func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { for key, value := range encryptionConfigSecret.Data { // Not all data keys are plugin configs — the Secret also contains the // encryption-config entry, so skip keys that don't match the pattern. - keyID, found, err := kms.KeyIDFromPluginConfigSecretDataKey(key) + keyID, found, err := keyIDFromPluginConfigSecretDataKey(key) if err != nil { return nil, fmt.Errorf("failed to extract keyID from data key %s: %w", key, err) } @@ -56,7 +84,26 @@ func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { kmsPlugins[keyID] = pluginConfig } - return &Config{Encryption: encryptionConfig, KMSPlugins: kmsPlugins}, nil + // Extract secret data entries from the encryption-config Secret. + // Data keys follow the format "kms-plugin-secret-{secretName}_{dataKey}-{keyID}" + // (e.g. "kms-plugin-secret-app-role_role-id-1"). keyIDFromSecretDataKey + // returns the keyID (e.g. "1") and the combined key (e.g. "app-role_role-id"), + // which is then split on "_" to recover secretName and dataKey. + var kmsPluginsSecretData KMSPluginsSecretData + for key, value := range encryptionConfigSecret.Data { + keyID, rawKey, found, err := parseSecretDataKey(key) + if err != nil { + return nil, fmt.Errorf("failed to parse secret data key %s: %w", key, err) + } + if !found { + continue + } + if err := kmsPluginsSecretData.SetFromRawKey(keyID, rawKey, value); err != nil { + return nil, fmt.Errorf("failed to set key %s: %w", key, err) + } + } + + return &Config{Encryption: encryptionConfig, KMSPlugins: kmsPlugins, KMSPluginsSecretData: kmsPluginsSecretData}, nil } func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { @@ -93,13 +140,22 @@ func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { if err != nil { return nil, fmt.Errorf("failed to encode KMS plugin config for key %s: %w", keyID, err) } - dataKey, err := kms.ToPluginConfigSecretDataKeyFor(keyID) + dataKey, err := toPluginConfigSecretDataKeyFor(keyID) if err != nil { return nil, err } s.Data[dataKey] = encodedPlugin } + // Write secret data entries to the encryption-config Secret. + // Each entry from FlatEntries() (e.g. "app-role_role-id") is combined with the keyID + // (e.g. "1") to produce "kms-plugin-secret-app-role_role-id-1". + for keyID, flatEntries := range secretData.KMSPluginsSecretData.FlatEntriesByKeyID() { + for flatKey, value := range flatEntries { + s.Data[encryptionConfigSecretDataPrefix+flatKey+"-"+keyID] = value + } + } + return s, nil } @@ -145,3 +201,19 @@ func ExtractUniqueAndSortedKMSConfigurations(secretData *Config) ([]*apiserverco }) return result, nil } + +func parseSecretDataKey(dataKey string) (keyID, rawKey string, found bool, err error) { + rest, found := strings.CutPrefix(dataKey, encryptionConfigSecretDataPrefix) + if !found { + return "", "", false, nil + } + i := strings.LastIndex(rest, "-") + if i < 1 { + return "", "", false, nil + } + keyID = rest[i+1:] + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return keyID, rest[:i], true, nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/helpers.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/helpers.go deleted file mode 100644 index 8a87f83e6b..0000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/helpers.go +++ /dev/null @@ -1,92 +0,0 @@ -package kms - -import ( - "fmt" - "strconv" - "strings" - - "github.com/openshift/api/features" - "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" - corev1 "k8s.io/api/core/v1" -) - -const pluginConfigDataKeyPrefix = "kms-plugin-config-" - -// ToPluginConfigSecretDataKeyFor constructs the data key for storing a KMS plugin config in the encryption-config Secret. -// The keyID must be a valid non-negative integer string. -func ToPluginConfigSecretDataKeyFor(keyID string) (string, error) { - if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { - return "", fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) - } - return pluginConfigDataKeyPrefix + keyID, nil -} - -// KeyIDFromPluginConfigSecretDataKey extracts the keyID from a kms-plugin-config data key. -// Returns the keyID and true if the key matches the "kms-plugin-config-" pattern. -func KeyIDFromPluginConfigSecretDataKey(dataKey string) (string, bool, error) { - keyID, found := strings.CutPrefix(dataKey, pluginConfigDataKeyPrefix) - if !found || len(keyID) == 0 { - return "", false, nil - } - if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { - return "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) - } - return keyID, true, nil -} - -// AddKMSPluginVolumeAndMountToPodSpec conditionally adds the KMS plugin volume mount to the specified container. -// It assumes the pod spec does not already contain the KMS volume or mount; no deduplication is performed. -// Deprecated: this is a temporary solution to get KMS TP v1 out. We should come up with a different approach afterwards. -func AddKMSPluginVolumeAndMountToPodSpec(podSpec *corev1.PodSpec, containerName string, featureGateAccessor featuregates.FeatureGateAccess) error { - if podSpec == nil { - return fmt.Errorf("pod spec cannot be nil") - } - - if !featureGateAccessor.AreInitialFeatureGatesObserved() { - return nil - } - - featureGates, err := featureGateAccessor.CurrentFeatureGates() - if err != nil { - return fmt.Errorf("failed to get feature gates: %w", err) - } - - if !featureGates.Enabled(features.FeatureGateKMSEncryption) { - return nil - } - - containerIndex := -1 - for i, container := range podSpec.Containers { - if container.Name == containerName { - containerIndex = i - break - } - } - - if containerIndex < 0 { - return fmt.Errorf("container %s not found", containerName) - } - - container := &podSpec.Containers[containerIndex] - container.VolumeMounts = append(container.VolumeMounts, - corev1.VolumeMount{ - Name: "kms-plugin-socket", - MountPath: "/var/run/kmsplugin", - }, - ) - - directoryOrCreate := corev1.HostPathDirectoryOrCreate - podSpec.Volumes = append(podSpec.Volumes, - corev1.Volume{ - Name: "kms-plugin-socket", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/run/kmsplugin", - Type: &directoryOrCreate, - }, - }, - }, - ) - - return nil -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go index e2e59efd6d..341cabbd80 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -76,16 +77,25 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { // encryption mode. return state.KeyState{}, fmt.Errorf("%s can not be empty, when mode is KMS", EncryptionSecretKMSEncryptionConfig) } - if v, ok := s.Data[EncryptionSecretKMSPluginConfig]; ok && len(v) > 0 { + if v, ok := s.Data[encryptionSecretKMSPluginConfig]; ok && len(v) > 0 { kmsConfig, err := encoding.DecodeKMSPluginConfig(v) if err != nil { - return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid %s data: %w", s.Namespace, s.Name, EncryptionSecretKMSPluginConfig, err) + return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid %s data: %w", s.Namespace, s.Name, encryptionSecretKMSPluginConfig, err) } key.KMS.Plugin = kmsConfig } else { // encryption.apiserver.operator.openshift.io-kms-plugin-config data field is required for KMS // encryption mode. - return state.KeyState{}, fmt.Errorf("%s can not be empty, when mode is KMS", EncryptionSecretKMSPluginConfig) + return state.KeyState{}, fmt.Errorf("%s can not be empty, when mode is KMS", encryptionSecretKMSPluginConfig) + } + for dataKey, value := range s.Data { + rawKey, found := strings.CutPrefix(dataKey, encryptionSecretKMSSecretDataPrefix) + if !found { + continue + } + if err := key.KMS.PluginSecretData.SetFromRawKey(rawKey, value); err != nil { + return state.KeyState{}, fmt.Errorf("secret %s/%s has malformed secret data key %q: %w", s.Namespace, s.Name, dataKey, err) + } } key.Mode = keyMode default: @@ -106,7 +116,7 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { } if ks.Mode == state.KMS && (!ks.HasKMSEncryption() || !ks.HasKMSPlugin()) { - return nil, fmt.Errorf("%s or %s can not be empty, when mode is KMS", EncryptionSecretKMSEncryptionConfig, EncryptionSecretKMSPluginConfig) + return nil, fmt.Errorf("%s or %s can not be empty, when mode is KMS", EncryptionSecretKMSEncryptionConfig, encryptionSecretKMSPluginConfig) } s := &corev1.Secret{ @@ -156,7 +166,13 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { if err != nil { return nil, err } - s.Data[EncryptionSecretKMSPluginConfig] = pluginData + s.Data[encryptionSecretKMSPluginConfig] = pluginData + } + + if ks.HasKMSSecretData() { + for flatKey, value := range ks.KMS.PluginSecretData.FlatEntries() { + s.Data[encryptionSecretKMSSecretDataPrefix+flatKey] = value + } } return s, nil diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go index 2795afbc0d..bf152dee31 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go @@ -54,9 +54,14 @@ const ( // encryption configuration for KMS mode in the encryption-key secret. EncryptionSecretKMSEncryptionConfig = "encryption.apiserver.operator.openshift.io-kms-encryption-config" - // EncryptionSecretKMSPluginConfig is the data field key that stores the serialized KMS plugin + // encryptionSecretKMSPluginConfig is the data field key that stores the serialized KMS plugin // configuration for KMS mode in the encryption-key secret. - EncryptionSecretKMSPluginConfig = "encryption.apiserver.operator.openshift.io-kms-plugin-config" + encryptionSecretKMSPluginConfig = "encryption.apiserver.operator.openshift.io-kms-plugin-config" + + // encryptionSecretKMSSecretDataPrefix is the data field key prefix for secret data values + // fetched from the referenced secret in openshift-config. The full data key is + // constructed as prefix + secretName + separator + dataKey. + encryptionSecretKMSSecretDataPrefix = "encryption.apiserver.operator.openshift.io-kms-plugin-secret-" ) // MigratedGroupResources is the data structured stored in the diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go index acaf493377..b7163c41e7 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go @@ -1,6 +1,8 @@ package state import ( + "fmt" + "strings" "time" configv1 "github.com/openshift/api/config/v1" @@ -15,6 +17,11 @@ const ( KubernetesDescriptionScaryValue = `WARNING: DO NOT EDIT. Altering of the encryption secrets will render you cluster inaccessible. Catastrophic data loss can occur from the most minor changes.` + + // secretDataKeySeparator separates the secret name from the data key. + // Underscore is used because it is forbidden in Kubernetes secret/configmap + // names, preventing collisions. + secretDataKeySeparator = "_" ) // GroupResourceState represents, for a single group resource, the write and read keys in a @@ -53,6 +60,10 @@ func (k *KeyState) HasKMSPlugin() bool { return k != nil && k.KMS != nil && k.KMS.Plugin != (configv1.KMSPluginConfig{}) } +func (k *KeyState) HasKMSSecretData() bool { + return k != nil && k.KMS != nil && len(k.KMS.PluginSecretData.Entries) > 0 +} + // KMSState stores all KMS encryption mode related configurations type KMSState struct { // Encoded EncryptionConfig that stores the KMS related fields @@ -60,6 +71,58 @@ type KMSState struct { // Plugin stores KMS plugin specific configurations Plugin configv1.KMSPluginConfig + + // PluginSecretData stores data key-value pairs fetched from referenced secrets. + PluginSecretData KMSSecretData +} + +// KMSSecretData stores data key-value pairs fetched from referenced secrets. +// Entries maps secret names to their data key-value pairs. +type KMSSecretData struct { + Entries map[string]map[string][]byte +} + +func (d *KMSSecretData) Set(secretName, dataKey string, value []byte) error { + if len(secretName) == 0 || len(dataKey) == 0 || len(value) == 0 { + return fmt.Errorf("secretName, dataKey, and value must not be empty") + } + if strings.Contains(secretName, "_") { + return fmt.Errorf("secret name %q must not contain underscores", secretName) + } + if d.Entries == nil { + d.Entries = map[string]map[string][]byte{} + } + if d.Entries[secretName] == nil { + d.Entries[secretName] = map[string][]byte{} + } + d.Entries[secretName][dataKey] = value + return nil +} + +// SetFromRawKey splits a combined key of the form "secretName_dataKey" +// and stores the value. +func (d *KMSSecretData) SetFromRawKey(rawKey string, value []byte) error { + parts := strings.SplitN(rawKey, secretDataKeySeparator, 2) + if len(parts) != 2 { + return fmt.Errorf("invalid combined key %q: expected format {secretName}%s{dataKey}", rawKey, secretDataKeySeparator) + } + return d.Set(parts[0], parts[1], value) +} + +// FlatEntries returns the stored data as a flat map keyed by "secretName_dataKey". +// "_" separates secretName from dataKey because "_" is forbidden in +// Kubernetes secret names, making the split unambiguous. +func (d *KMSSecretData) FlatEntries() map[string][]byte { + if d.Entries == nil { + return nil + } + result := map[string][]byte{} + for secretName, keys := range d.Entries { + for dataKey, value := range keys { + result[secretName+secretDataKeySeparator+dataKey] = value + } + } + return result } type MigrationState struct { diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go b/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go index b32bb13f84..329fc0e98e 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go @@ -56,9 +56,8 @@ type EncryptionKeyMeta struct { type UpdateUnsupportedConfigFunc func(raw []byte) error -func SetAndWaitForEncryptionType(t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { +func SetAndWaitForEncryptionType(ctx context.Context, t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { t.Helper() - ctx := context.TODO() t.Logf("Starting encryption e2e test for %q mode", provider.Type) @@ -347,3 +346,34 @@ func setUpTearDown(namespace string) func(testing.TB, bool) { } } } + +// WaitForPodImagePullBackOff polls pods in the given namespace until at least one pod +// has an init container or container stuck in ImagePullBackOff. This is useful for +// detecting that an invalid KMS plugin image is causing static pod failure. +func WaitForPodImagePullBackOff(ctx context.Context, t testing.TB, kubeClient kubernetes.Interface, namespace, labelSelector string, timeout time.Duration) { + t.Helper() + t.Logf("Waiting up to %s for a pod in %s (selector=%s) to enter ImagePullBackOff", timeout, namespace, labelSelector) + err := wait.PollUntilContextTimeout(ctx, waitPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + t.Logf("Error listing pods: %v", err) + return false, nil + } + for _, pod := range pods.Items { + for _, cs := range pod.Status.InitContainerStatuses { + if cs.State.Waiting != nil && (cs.State.Waiting.Reason == "ImagePullBackOff" || cs.State.Waiting.Reason == "ErrImagePull") { + t.Logf("Pod %s init container %s is in %s", pod.Name, cs.Name, cs.State.Waiting.Reason) + return true, nil + } + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && (cs.State.Waiting.Reason == "ImagePullBackOff" || cs.State.Waiting.Reason == "ErrImagePull") { + t.Logf("Pod %s container %s is in %s", pod.Name, cs.Name, cs.State.Waiting.Reason) + return true, nil + } + } + } + return false, nil + }) + require.NoError(t, err, "timed out waiting for pod to enter ImagePullBackOff in namespace %s", namespace) +} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go b/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go index 601002754d..2c44aefd0e 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go @@ -33,6 +33,11 @@ const ( defaultVaultTransitKey = "kms-key" defaultAppRoleTargetNamespace = "openshift-config" vaultCommandTimeout = 30 * time.Second + + // InvalidVaultKMSPluginImage is an OCI image reference that passes API validation + // (correct format, sha256 digest, sufficient length) but does not exist in any registry. + // Use this to test degradation when the KMS plugin image cannot be pulled. + InvalidVaultKMSPluginImage = "quay.io/openshifttest/mock-kms-plugin-nonexistent@sha256:0000000000000000000000000000000000000000000000000000000000000000" ) // DefaultVaultEncryptionProvider is a ready-to-use Vault KMS EncryptionProvider for e2e tests. @@ -42,6 +47,40 @@ var DefaultVaultEncryptionProvider = library.EncryptionProvider{ Setup: ensureDefaultVaultAppRoleSecret, } +var DefaultFakeVaultEncryptionProvider = library.EncryptionProvider{ + APIServerEncryption: DefaultFakeKMSPluginConfig, + Setup: ensureDefaultVaultAppRoleSecret, +} + +// InvalidImageVaultEncryptionProvider is a Vault KMS EncryptionProvider with a +// non-existent plugin image. Use this for testing degradation when the image cannot be pulled. +var InvalidImageVaultEncryptionProvider = library.EncryptionProvider{ + APIServerEncryption: InvalidImageVaultKMSPluginConfig, + Setup: ensureDefaultVaultAppRoleSecret, +} + +// InvalidImageVaultKMSPluginConfig is identical to DefaultVaultKMSPluginConfig +// but uses a non-existent image that will fail to pull. +var InvalidImageVaultKMSPluginConfig = configv1.APIServerEncryption{ + Type: configv1.EncryptionTypeKMS, + KMS: configv1.KMSPluginConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSPluginConfig{ + KMSPluginImage: InvalidVaultKMSPluginImage, + VaultAddress: defaultVaultAddress, + VaultNamespace: defaultVaultEnterpriseNS, + TransitMount: defaultVaultTransitMount, + TransitKey: defaultVaultTransitKey, + Authentication: configv1.VaultAuthentication{ + Type: configv1.VaultAuthenticationTypeAppRole, + AppRole: configv1.VaultAppRoleAuthentication{ + Secret: configv1.VaultSecretReference{Name: defaultVaultAppRoleSecretName}, + }, + }, + }, + }, +} + // DefaultVaultKMSPluginConfig is the standard Vault KMS encryption config // used by CI e2e tests. var DefaultVaultKMSPluginConfig = configv1.APIServerEncryption{ @@ -65,18 +104,22 @@ var DefaultVaultKMSPluginConfig = configv1.APIServerEncryption{ } // DefaultFakeKMSPluginConfig is a fake Vault KMS configuration used by unit tests. -var DefaultFakeKMSPluginConfig = configv1.KMSPluginConfig{ - Type: configv1.VaultKMSProvider, - Vault: configv1.VaultKMSPluginConfig{ - KMSPluginImage: WellKnownUpstreamMockKMSPluginImage, - VaultAddress: "https://vault.example.com", - Authentication: configv1.VaultAuthentication{ - Type: configv1.VaultAuthenticationTypeAppRole, - AppRole: configv1.VaultAppRoleAuthentication{ - Secret: configv1.VaultSecretReference{Name: "vault-approle-secret"}, +var DefaultFakeKMSPluginConfig = configv1.APIServerEncryption{ + Type: configv1.EncryptionTypeKMS, + KMS: configv1.KMSPluginConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSPluginConfig{ + KMSPluginImage: WellKnownUpstreamMockKMSPluginImage, + VaultAddress: "https://vault.example.com", + Authentication: configv1.VaultAuthentication{ + Type: configv1.VaultAuthenticationTypeAppRole, + AppRole: configv1.VaultAppRoleAuthentication{ + Secret: configv1.VaultSecretReference{Name: defaultVaultAppRoleSecretName}, + }, }, + TransitKey: "test-transit-key", + TransitMount: defaultVaultTransitMount, }, - TransitKey: "test-transit-key", }, } @@ -97,8 +140,8 @@ func ensureDefaultVaultAppRoleSecret(ctx context.Context, t testing.TB) { }, Type: corev1.SecretTypeOpaque, Data: map[string][]byte{ - "roleID": creds.Data["role-id"], - "secretID": creds.Data["secret-id"], + "role-id": creds.Data["role-id"], + "secret-id": creds.Data["secret-id"], }, } recorder := events.NewInMemoryRecorder("vault-approle-secret-setup", clock.RealClock{}) diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go b/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go index c45a28e26e..9e3a7cebfd 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go @@ -1,6 +1,7 @@ package encryption import ( + "context" "testing" "time" @@ -22,13 +23,13 @@ type PerfScenario struct { EncryptionProvider EncryptionProvider } -func TestPerfEncryption(t *testing.T, scenario PerfScenario) { +func TestPerfEncryption(ctx context.Context, t *testing.T, scenario PerfScenario) { e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) migrationStartedCh := make(chan time.Time, 1) populateDatabase(e, scenario.DBLoaderWorkers, scenario.DBLoaderFunc, scenario.AssertDBPopulatedFunc) watchForMigrationControllerProgressingConditionAsync(e, scenario.GetOperatorConditionsFunc, migrationStartedCh) - endTimeStamp := runTestEncryption(t, scenario) + endTimeStamp := runTestEncryption(ctx, t, scenario) select { case migrationStarted := <-migrationStartedCh: @@ -38,9 +39,9 @@ func TestPerfEncryption(t *testing.T, scenario PerfScenario) { } } -func runTestEncryption(tt *testing.T, scenario PerfScenario) time.Time { +func runTestEncryption(ctx context.Context, tt *testing.T, scenario PerfScenario) time.Time { var ts time.Time - TestEncryptionType(tt, BasicScenario{ + TestEncryptionType(ctx, tt, BasicScenario{ Namespace: scenario.Namespace, LabelSelector: scenario.LabelSelector, EncryptionConfigSecretName: scenario.EncryptionConfigSecretName, diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go index 8d8c62c937..27bb057519 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/rand" configv1 "github.com/openshift/api/config/v1" ) @@ -31,20 +31,18 @@ type BasicScenario struct { type EncryptionProvider struct { configv1.APIServerEncryption // Setup is called once before the provider is first used. May be nil. - // Context is accepted as an explicit argument because testing.TB.Context() - // is not supported by all implementations (e.g. Ginkgo's GinkgoTBWrapper). Setup func(ctx context.Context, t testing.TB) } -func TestEncryptionTypeIdentity(t testing.TB, scenario BasicScenario) { +func TestEncryptionTypeIdentity(ctx context.Context, t testing.TB, scenario BasicScenario) { e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeIdentity}}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(ctx, e, EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeIdentity}}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, configv1.EncryptionTypeIdentity, scenario.Namespace, scenario.LabelSelector) } -func TestEncryptionTypeUnset(t testing.TB, scenario BasicScenario) { +func TestEncryptionTypeUnset(ctx context.Context, t testing.TB, scenario BasicScenario) { e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, EncryptionProvider{}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(ctx, e, EncryptionProvider{}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, configv1.EncryptionTypeIdentity, scenario.Namespace, scenario.LabelSelector) } @@ -59,40 +57,40 @@ func resolveProvider(t testing.TB, defaultType configv1.EncryptionType, provider return EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: defaultType}} } -func TestEncryptionTypeAESCBC(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { +func TestEncryptionTypeAESCBC(ctx context.Context, t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeAESCBC, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(ctx, e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, provider.Type, scenario.Namespace, scenario.LabelSelector) AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionTypeAESGCM(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { +func TestEncryptionTypeAESGCM(ctx context.Context, t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeAESGCM, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(ctx, e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, provider.Type, scenario.Namespace, scenario.LabelSelector) AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionTypeKMS(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { +func TestEncryptionTypeKMS(ctx context.Context, t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeKMS, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(ctx, e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, provider.Type, scenario.Namespace, scenario.LabelSelector) AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionType(t testing.TB, scenario BasicScenario, provider EncryptionProvider) { +func TestEncryptionType(ctx context.Context, t testing.TB, scenario BasicScenario, provider EncryptionProvider) { switch provider.Type { case configv1.EncryptionTypeAESCBC: - TestEncryptionTypeAESCBC(t, scenario, provider) + TestEncryptionTypeAESCBC(ctx, t, scenario, provider) case configv1.EncryptionTypeAESGCM: - TestEncryptionTypeAESGCM(t, scenario, provider) + TestEncryptionTypeAESGCM(ctx, t, scenario, provider) case configv1.EncryptionTypeKMS: - TestEncryptionTypeKMS(t, scenario, provider) + TestEncryptionTypeKMS(ctx, t, scenario, provider) case configv1.EncryptionTypeIdentity, "": - TestEncryptionTypeIdentity(t, scenario) + TestEncryptionTypeIdentity(ctx, t, scenario) default: t.Fatalf("Unknown encryption type: %s", provider.Type) } @@ -113,28 +111,28 @@ type testStep struct { testFunc func(testing.TB) } -func TestEncryptionTurnOnAndOff(t testing.TB, scenario OnOffScenario) { +func TestEncryptionTurnOnAndOff(ctx context.Context, t testing.TB, scenario OnOffScenario) { scenarios := []testStep{ {name: fmt.Sprintf("CreateAndStore%s", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.CreateResourceFunc(e, GetClients(e), scenario.Namespace) }}, - {name: fmt.Sprintf("On%s", strings.ToUpper(string(scenario.EncryptionProvider.Type))), testFunc: func(t testing.TB) { TestEncryptionType(t, scenario.BasicScenario, scenario.EncryptionProvider) }}, + {name: fmt.Sprintf("On%s", strings.ToUpper(string(scenario.EncryptionProvider.Type))), testFunc: func(t testing.TB) { TestEncryptionType(ctx, t, scenario.BasicScenario, scenario.EncryptionProvider) }}, {name: fmt.Sprintf("Assert%sEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.AssertResourceEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) }}, - {name: "OffIdentity", testFunc: func(t testing.TB) { TestEncryptionTypeIdentity(t, scenario.BasicScenario) }}, + {name: "OffIdentity", testFunc: func(t testing.TB) { TestEncryptionTypeIdentity(ctx, t, scenario.BasicScenario) }}, {name: fmt.Sprintf("Assert%sNotEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) }}, - {name: fmt.Sprintf("On%sSecond", strings.ToUpper(string(scenario.EncryptionProvider.Type))), testFunc: func(t testing.TB) { TestEncryptionType(t, scenario.BasicScenario, scenario.EncryptionProvider) }}, + {name: fmt.Sprintf("On%sSecond", strings.ToUpper(string(scenario.EncryptionProvider.Type))), testFunc: func(t testing.TB) { TestEncryptionType(ctx, t, scenario.BasicScenario, scenario.EncryptionProvider) }}, {name: fmt.Sprintf("Assert%sEncryptedSecond", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.AssertResourceEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) }}, - {name: "OffIdentitySecond", testFunc: func(t testing.TB) { TestEncryptionTypeIdentity(t, scenario.BasicScenario) }}, + {name: "OffIdentitySecond", testFunc: func(t testing.TB) { TestEncryptionTypeIdentity(ctx, t, scenario.BasicScenario) }}, {name: fmt.Sprintf("Assert%sNotEncryptedSecond", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) @@ -184,7 +182,7 @@ func ShuffleEncryptionProviders(providers []EncryptionProvider) []EncryptionProv // It creates a resource, migrates through each provider, // verifies the resource is encrypted after each migration, and finally // switches to identity (off). -func TestEncryptionProvidersMigration(t testing.TB, scenario ProvidersMigrationScenario) { +func TestEncryptionProvidersMigration(ctx context.Context, t testing.TB, scenario ProvidersMigrationScenario) { if len(scenario.EncryptionProviders) < 2 { t.Fatalf("ProvidersMigrationScenario requires at least 2 encryption providers, got %d", len(scenario.EncryptionProviders)) } @@ -211,7 +209,7 @@ func TestEncryptionProvidersMigration(t testing.TB, scenario ProvidersMigrationS } scenarios = append(scenarios, testStep{name: fmt.Sprintf("%s%s", prefix, strings.ToUpper(string(provider.Type))), testFunc: func(t testing.TB) { - TestEncryptionType(t, scenario.BasicScenario, provider) + TestEncryptionType(ctx, t, scenario.BasicScenario, provider) }}, testStep{name: fmt.Sprintf("Assert%sEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) @@ -222,7 +220,7 @@ func TestEncryptionProvidersMigration(t testing.TB, scenario ProvidersMigrationS // step 3: switch to identity (off) to verify the resource is re-written unencrypted scenarios = append(scenarios, testStep{name: fmt.Sprintf("OffIdentityAndAssert%sNotEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { - TestEncryptionTypeIdentity(t, scenario.BasicScenario) + TestEncryptionTypeIdentity(ctx, t, scenario.BasicScenario) e := NewE(t) scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) }}) @@ -248,7 +246,7 @@ type RotationScenario struct { // TestEncryptionRotation first encrypts data with aescbc key // then it forces a key rotation by setting the "encyrption.Reason" in the operator's configuration file -func TestEncryptionRotation(t testing.TB, scenario RotationScenario) { +func TestEncryptionRotation(ctx context.Context, t testing.TB, scenario RotationScenario) { // test data ns := scenario.Namespace labelSelector := scenario.LabelSelector @@ -259,7 +257,7 @@ func TestEncryptionRotation(t testing.TB, scenario RotationScenario) { scenario.CreateResourceFunc(e, GetClients(e), ns) // step 2: run provided encryption scenario - TestEncryptionType(t, scenario.BasicScenario, scenario.EncryptionProvider) + TestEncryptionType(ctx, t, scenario.BasicScenario, scenario.EncryptionProvider) // step 3: take samples rawEncryptedResourceWithKey1 := scenario.GetRawResourceFunc(e, clientSet, ns) @@ -279,3 +277,70 @@ func TestEncryptionRotation(t testing.TB, scenario RotationScenario) { // TODO: assert conditions - operator and encryption migration controller must report status as active not progressing, and not failing for all scenarios } + +// InvalidImageRecoveryScenario tests that an invalid KMS plugin image causes +// the operator to degrade (even when switching to aescbc) and that providing +// a valid KMS image restores normal operation. +// +// Flow: +// 1. Enable KMS with invalid (non-existent) plugin image +// 2. Switch to aescbc — cluster should remain stuck on KMS (rollout blocked) +// 3. Operator reports degraded (stuck NodeInstaller, ImagePullBackOff, etc.) +// 4. Fix by applying KMS with valid image and wait for full encryption migration +type InvalidImageRecoveryScenario struct { + BasicScenario + // InvalidImageProvider is the KMS EncryptionProvider configured with an invalid + // (non-existent or broken) KMS plugin image. Enabling this should cause degradation. + InvalidImageProvider EncryptionProvider + // ValidImageProvider is the KMS EncryptionProvider configured with the correct + // KMS plugin image. Applying this after degradation should restore the cluster. + ValidImageProvider EncryptionProvider + // WaitForDegraded should block until the operator reports a degraded condition. + WaitForDegraded func(ctx context.Context, t testing.TB) +} + +// TestEncryptionInvalidImageRecovery tests that: +// 1. Enabling KMS with an invalid plugin image causes the operator to degrade +// 2. Switching to aescbc does NOT recover (cluster is stuck on the failed KMS rollout) +// 3. Providing a valid KMS image restores the cluster (uses SetAndWaitForEncryptionType) +func TestEncryptionInvalidImageRecovery(ctx context.Context, t testing.TB, scenario InvalidImageRecoveryScenario) { + e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) + require.NotNil(t, scenario.WaitForDegraded, "WaitForDegraded must not be nil") + require.Equal(t, configv1.EncryptionTypeKMS, scenario.InvalidImageProvider.Type, "InvalidImageProvider must be KMS type") + require.Equal(t, configv1.EncryptionTypeKMS, scenario.ValidImageProvider.Type, "ValidImageProvider must be KMS type") + + cs := GetClients(e) + + // step 1: enable KMS with invalid image (set config but don't wait for completion) + t.Log("Setting KMS encryption with invalid plugin image") + if scenario.InvalidImageProvider.Setup != nil { + scenario.InvalidImageProvider.Setup(ctx, e) + } + apiServer, err := cs.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err) + apiServer.Spec.Encryption = scenario.InvalidImageProvider.APIServerEncryption + _, err = cs.ApiServerConfig.Update(ctx, apiServer, metav1.UpdateOptions{}) + require.NoError(t, err) + + // step 2: attempt to switch to aescbc — the cluster should remain stuck on KMS + // because the static pod rollout with the invalid image is blocking progress + t.Log("Attempting to switch to aescbc (cluster should remain stuck on KMS)") + apiServer, err = cs.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err) + apiServer.Spec.Encryption = configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC} + _, err = cs.ApiServerConfig.Update(ctx, apiServer, metav1.UpdateOptions{}) + require.NoError(t, err) + + // step 3: wait for degraded — the operator is stuck because the invalid KMS image + // prevents the NodeInstaller from completing the rollout + t.Log("Waiting for operator to report degraded status") + scenario.WaitForDegraded(ctx, e) + + // step 4: fix by applying KMS with valid image and wait for full recovery + // (uses SetAndWaitForEncryptionType which waits for encryption key migration, + // same as the standard encryption on/off and migration tests) + t.Log("Recovering: applying KMS with valid plugin image and waiting for full migration") + SetAndWaitForEncryptionType(ctx, e, scenario.ValidImageProvider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + + t.Log("Invalid image recovery test passed") +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 172bd78f11..3286729d57 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -407,7 +407,7 @@ github.com/openshift/client-go/security/informers/externalversions/internalinter github.com/openshift/client-go/security/informers/externalversions/security github.com/openshift/client-go/security/informers/externalversions/security/v1 github.com/openshift/client-go/security/listers/security/v1 -# github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9 +# github.com/openshift/library-go v0.0.0-20260520123929-8dbb42ebf1e9 => github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86 ## explicit; go 1.25.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/assets @@ -445,7 +445,6 @@ github.com/openshift/library-go/pkg/operator/encryption/crypto github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata -github.com/openshift/library-go/pkg/operator/encryption/kms github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets @@ -1700,3 +1699,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 +# github.com/openshift/library-go => github.com/gangwgr/library-go v0.0.0-20260527082938-d94ed6169d86