Skip to content

Commit 0d54a40

Browse files
muraeeclaude
muraee
and
claude
committed
feat: add etcd data re-encryption after encryption key rotation
Add a re-encryption controller in the HCCO that triggers StorageVersionMigration after an encryption key rotation, ensuring all existing etcd data is re-encrypted with the new active key. Components: - API: EtcdDataEncryptionUpToDate condition type and reasons - CPO: key fingerprint computation and rekey-needed annotation on kas-secret-encryption-config secret - HCCO: new reencryption controller using library-go's KubeStorageVersionMigrator to drive StorageVersionMigration CRs - HyperShift Operator: condition bubble-up from HCP to HostedCluster Ref: OCPSTRAT-2527, OCPSTRAT-2540 Enhancement: openshift/enhancements#1969 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 8fb1c57 commit 0d54a40

9 files changed

Lines changed: 1028 additions & 0 deletions

File tree

‎api/hypershift/v1beta1/hostedcluster_conditions.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,13 @@ const (
243243
// **False / AutoNodeProgressing** means AutoNode is being enabled or disabled — the operation is in progress.
244244
// **False / AutoNodeNotConfigured** means AutoNode is not configured in the spec and all Karpenter components have been removed.
245245
AutoNodeEnabled ConditionType = "AutoNodeEnabled"
246+
247+
// EtcdDataEncryptionUpToDate indicates whether all etcd data is encrypted
248+
// with the currently active encryption key.
249+
// **True** means all data has been confirmed encrypted with the active key.
250+
// **False** means re-encryption is in progress or has failed.
251+
// Absent: encryption is not configured or no key rotation has occurred.
252+
EtcdDataEncryptionUpToDate ConditionType = "EtcdDataEncryptionUpToDate"
246253
)
247254

248255
// Reasons.
@@ -322,6 +329,11 @@ const (
322329
AutoNodeNotConfiguredReason = "AutoNodeNotConfigured"
323330
AutoNodeProgressingReason = "AutoNodeProgressing"
324331
AutoNodeEvaluationFailedReason = "AutoNodeEvaluationFailed"
332+
333+
ReEncryptionInProgressReason = "ReEncryptionInProgress"
334+
ReEncryptionCompletedReason = "ReEncryptionCompleted"
335+
ReEncryptionFailedReason = "ReEncryptionFailed"
336+
ReEncryptionWaitingForKAS = "WaitingForKASConvergence"
325337
)
326338

327339
// Messages.

‎control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package kas
22

33
import (
4+
"crypto/sha256"
45
"fmt"
56

67
hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
@@ -21,6 +22,14 @@ const (
2122
encryptionConfigurationKind = "EncryptionConfiguration"
2223

2324
secretEncryptionConfigFileVolumeName = "kas-secret-encryption-config"
25+
26+
// EncryptionActiveKeyHashAnnotation stores a SHA-256 fingerprint of the active
27+
// encryption key identity on the kas-secret-encryption-config secret.
28+
EncryptionActiveKeyHashAnnotation = "hypershift.openshift.io/encryption-active-key-hash"
29+
// EncryptionRekeyNeededAnnotation is set to "true" on the kas-secret-encryption-config
30+
// secret when the active key changes, signaling the HCCO re-encryption controller to
31+
// create StorageVersionMigration CRs. It is removed after re-encryption completes.
32+
EncryptionRekeyNeededAnnotation = "hypershift.openshift.io/encryption-rekey-needed"
2433
)
2534

2635
func secretEncryptionConfigPredicate(cpContext component.WorkloadContext) bool {
@@ -85,6 +94,23 @@ func adaptSecretEncryptionConfig(cpContext component.WorkloadContext, secret *co
8594
}
8695

8796
secret.Data[secretEncryptionConfigurationKey] = data
97+
98+
// Compute the active key fingerprint and detect key changes.
99+
fingerprint, err := computeActiveKeyFingerprint(cpContext)
100+
if err != nil {
101+
return fmt.Errorf("failed to compute active key fingerprint: %w", err)
102+
}
103+
if fingerprint != "" {
104+
if secret.Annotations == nil {
105+
secret.Annotations = map[string]string{}
106+
}
107+
existingHash := secret.Annotations[EncryptionActiveKeyHashAnnotation]
108+
if existingHash != fingerprint {
109+
secret.Annotations[EncryptionActiveKeyHashAnnotation] = fingerprint
110+
secret.Annotations[EncryptionRekeyNeededAnnotation] = "true"
111+
}
112+
}
113+
88114
return nil
89115
}
90116

@@ -120,6 +146,64 @@ func getKMSAPIVersion(cpContext component.WorkloadContext, secret *corev1.Secret
120146
return apiVersion, nil
121147
}
122148

149+
// computeActiveKeyFingerprint computes a SHA-256 fingerprint of the active
150+
// encryption key identity. The fingerprint is used to detect key changes.
151+
func computeActiveKeyFingerprint(cpContext component.WorkloadContext) (string, error) {
152+
secretEncryption := cpContext.HCP.Spec.SecretEncryption
153+
if secretEncryption == nil {
154+
return "", nil
155+
}
156+
157+
var input string
158+
switch secretEncryption.Type {
159+
case hyperv1.KMS:
160+
if secretEncryption.KMS == nil {
161+
return "", nil
162+
}
163+
switch secretEncryption.KMS.Provider {
164+
case hyperv1.AZURE:
165+
if secretEncryption.KMS.Azure == nil {
166+
return "", nil
167+
}
168+
key := secretEncryption.KMS.Azure.ActiveKey
169+
input = fmt.Sprintf("azure/%s/%s/%s", key.KeyVaultName, key.KeyName, key.KeyVersion)
170+
case hyperv1.AWS:
171+
if secretEncryption.KMS.AWS == nil {
172+
return "", nil
173+
}
174+
input = fmt.Sprintf("aws/%s", secretEncryption.KMS.AWS.ActiveKey.ARN)
175+
case hyperv1.IBMCloud:
176+
if secretEncryption.KMS.IBMCloud == nil || len(secretEncryption.KMS.IBMCloud.KeyList) == 0 {
177+
return "", nil
178+
}
179+
key := secretEncryption.KMS.IBMCloud.KeyList[0]
180+
input = fmt.Sprintf("ibmcloud/%s/%d", key.CRKID, key.KeyVersion)
181+
default:
182+
return "", fmt.Errorf("unsupported KMS provider: %s", secretEncryption.KMS.Provider)
183+
}
184+
case hyperv1.AESCBC:
185+
if secretEncryption.AESCBC == nil || len(secretEncryption.AESCBC.ActiveKey.Name) == 0 {
186+
return "", nil
187+
}
188+
activeKeySecret := &corev1.Secret{
189+
ObjectMeta: metav1.ObjectMeta{
190+
Name: secretEncryption.AESCBC.ActiveKey.Name,
191+
Namespace: cpContext.HCP.Namespace,
192+
},
193+
}
194+
if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(activeKeySecret), activeKeySecret); err != nil {
195+
return "", fmt.Errorf("failed to get aescbc active key secret for fingerprint: %w", err)
196+
}
197+
keyData := activeKeySecret.Data[hyperv1.AESCBCKeySecretKey]
198+
input = fmt.Sprintf("aescbc/%s/%x", secretEncryption.AESCBC.ActiveKey.Name, sha256.Sum256(keyData))
199+
default:
200+
return "", nil
201+
}
202+
203+
hash := sha256.Sum256([]byte(input))
204+
return fmt.Sprintf("%x", hash), nil
205+
}
206+
123207
func buildVolumeSecretEncryptionConfigFile() corev1.Volume {
124208
v := corev1.Volume{
125209
Name: secretEncryptionConfigFileVolumeName,

‎control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package kas
22

33
import (
44
"bytes"
5+
"context"
56
"testing"
67
"time"
78

@@ -154,6 +155,198 @@ func TestReconcileKMSEncryptionConfigAWS(t *testing.T) {
154155
}
155156
}
156157

158+
func TestComputeActiveKeyFingerprint(t *testing.T) {
159+
testCases := []struct {
160+
name string
161+
secretEncryption *hyperv1.SecretEncryptionSpec
162+
objects []corev1.Secret
163+
expectFingerprint bool
164+
expectSameAs string // if set, compare against this test case's fingerprint
165+
expectDifferentKey bool // if true, fingerprint must differ from expectSameAs
166+
}{
167+
{
168+
name: "When encryption is nil it should return empty fingerprint",
169+
secretEncryption: nil,
170+
expectFingerprint: false,
171+
},
172+
{
173+
name: "When AWS KMS key is configured it should return a fingerprint",
174+
secretEncryption: &hyperv1.SecretEncryptionSpec{
175+
Type: hyperv1.KMS,
176+
KMS: &hyperv1.KMSSpec{
177+
Provider: hyperv1.AWS,
178+
AWS: &hyperv1.AWSKMSSpec{
179+
ActiveKey: hyperv1.AWSKMSKeyEntry{ARN: "arn:aws:kms:us-east-1:123456789:key/test-key-1"},
180+
},
181+
},
182+
},
183+
expectFingerprint: true,
184+
},
185+
{
186+
name: "When AWS KMS key changes it should return a different fingerprint",
187+
secretEncryption: &hyperv1.SecretEncryptionSpec{
188+
Type: hyperv1.KMS,
189+
KMS: &hyperv1.KMSSpec{
190+
Provider: hyperv1.AWS,
191+
AWS: &hyperv1.AWSKMSSpec{
192+
ActiveKey: hyperv1.AWSKMSKeyEntry{ARN: "arn:aws:kms:us-east-1:123456789:key/test-key-2"},
193+
},
194+
},
195+
},
196+
expectFingerprint: true,
197+
},
198+
{
199+
name: "When Azure KMS key is configured it should return a fingerprint",
200+
secretEncryption: &hyperv1.SecretEncryptionSpec{
201+
Type: hyperv1.KMS,
202+
KMS: &hyperv1.KMSSpec{
203+
Provider: hyperv1.AZURE,
204+
Azure: &hyperv1.AzureKMSSpec{
205+
ActiveKey: hyperv1.AzureKMSKey{
206+
KeyVaultName: "test-vault",
207+
KeyName: "test-key",
208+
KeyVersion: "v1",
209+
},
210+
},
211+
},
212+
},
213+
expectFingerprint: true,
214+
},
215+
{
216+
name: "When IBM Cloud KMS key is configured it should return a fingerprint",
217+
secretEncryption: &hyperv1.SecretEncryptionSpec{
218+
Type: hyperv1.KMS,
219+
KMS: &hyperv1.KMSSpec{
220+
Provider: hyperv1.IBMCloud,
221+
IBMCloud: &hyperv1.IBMCloudKMSSpec{
222+
KeyList: []hyperv1.IBMCloudKMSKeyEntry{
223+
{CRKID: "test-crk-id", KeyVersion: 1},
224+
},
225+
},
226+
},
227+
},
228+
expectFingerprint: true,
229+
},
230+
{
231+
name: "When AESCBC key is configured it should return a fingerprint",
232+
secretEncryption: &hyperv1.SecretEncryptionSpec{
233+
Type: hyperv1.AESCBC,
234+
AESCBC: &hyperv1.AESCBCSpec{
235+
ActiveKey: corev1.LocalObjectReference{Name: "aescbc-key"},
236+
},
237+
},
238+
objects: []corev1.Secret{
239+
{
240+
ObjectMeta: metav1.ObjectMeta{Name: "aescbc-key", Namespace: "test-namespace"},
241+
Data: map[string][]byte{hyperv1.AESCBCKeySecretKey: []byte("test-key-data")},
242+
},
243+
},
244+
expectFingerprint: true,
245+
},
246+
{
247+
name: "When KMS spec is nil it should return empty fingerprint",
248+
secretEncryption: &hyperv1.SecretEncryptionSpec{
249+
Type: hyperv1.KMS,
250+
KMS: nil,
251+
},
252+
expectFingerprint: false,
253+
},
254+
}
255+
256+
for _, tc := range testCases {
257+
t.Run(tc.name, func(t *testing.T) {
258+
clientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme)
259+
for i := range tc.objects {
260+
clientBuilder.WithObjects(&tc.objects[i])
261+
}
262+
263+
cpContext := controlplanecomponent.WorkloadContext{
264+
HCP: &hyperv1.HostedControlPlane{
265+
ObjectMeta: metav1.ObjectMeta{
266+
Namespace: "test-namespace",
267+
},
268+
Spec: hyperv1.HostedControlPlaneSpec{
269+
SecretEncryption: tc.secretEncryption,
270+
},
271+
},
272+
Client: clientBuilder.Build(),
273+
}
274+
cpContext.Context = context.Background()
275+
276+
fingerprint, err := computeActiveKeyFingerprint(cpContext)
277+
if err != nil {
278+
t.Fatalf("unexpected error: %v", err)
279+
}
280+
281+
if tc.expectFingerprint {
282+
if fingerprint == "" {
283+
t.Error("expected non-empty fingerprint")
284+
}
285+
// Verify determinism: same input produces same output.
286+
fingerprint2, err := computeActiveKeyFingerprint(cpContext)
287+
if err != nil {
288+
t.Fatalf("unexpected error on second call: %v", err)
289+
}
290+
if fingerprint != fingerprint2 {
291+
t.Errorf("fingerprint not deterministic: %s != %s", fingerprint, fingerprint2)
292+
}
293+
} else {
294+
if fingerprint != "" {
295+
t.Errorf("expected empty fingerprint, got %s", fingerprint)
296+
}
297+
}
298+
})
299+
}
300+
}
301+
302+
func TestComputeActiveKeyFingerprint_DifferentKeysProduceDifferentHashes(t *testing.T) {
303+
clientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme)
304+
key1Context := controlplanecomponent.WorkloadContext{
305+
HCP: &hyperv1.HostedControlPlane{
306+
Spec: hyperv1.HostedControlPlaneSpec{
307+
SecretEncryption: &hyperv1.SecretEncryptionSpec{
308+
Type: hyperv1.KMS,
309+
KMS: &hyperv1.KMSSpec{
310+
Provider: hyperv1.AWS,
311+
AWS: &hyperv1.AWSKMSSpec{ActiveKey: hyperv1.AWSKMSKeyEntry{ARN: "arn:key-1"}},
312+
},
313+
},
314+
},
315+
},
316+
Client: clientBuilder.Build(),
317+
}
318+
key1Context.Context = context.Background()
319+
320+
key2Context := controlplanecomponent.WorkloadContext{
321+
HCP: &hyperv1.HostedControlPlane{
322+
Spec: hyperv1.HostedControlPlaneSpec{
323+
SecretEncryption: &hyperv1.SecretEncryptionSpec{
324+
Type: hyperv1.KMS,
325+
KMS: &hyperv1.KMSSpec{
326+
Provider: hyperv1.AWS,
327+
AWS: &hyperv1.AWSKMSSpec{ActiveKey: hyperv1.AWSKMSKeyEntry{ARN: "arn:key-2"}},
328+
},
329+
},
330+
},
331+
},
332+
Client: clientBuilder.Build(),
333+
}
334+
key2Context.Context = context.Background()
335+
336+
fp1, err := computeActiveKeyFingerprint(key1Context)
337+
if err != nil {
338+
t.Fatalf("unexpected error: %v", err)
339+
}
340+
fp2, err := computeActiveKeyFingerprint(key2Context)
341+
if err != nil {
342+
t.Fatalf("unexpected error: %v", err)
343+
}
344+
345+
if fp1 == fp2 {
346+
t.Errorf("different keys produced same fingerprint: %s", fp1)
347+
}
348+
}
349+
157350
func generateExpectedEncryptionConfig(apiVersion string) *v1.EncryptionConfiguration {
158351
config := &v1.EncryptionConfiguration{
159352
TypeMeta: metav1.TypeMeta{Kind: "EncryptionConfiguration", APIVersion: "apiserver.config.k8s.io/v1"},

‎control-plane-operator/hostedclusterconfigoperator/api/scheme.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
2828

2929
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
30+
svmv1alpha1 "sigs.k8s.io/kube-storage-version-migrator/pkg/apis/migration/v1alpha1"
3031

3132
snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
3233
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -74,4 +75,5 @@ func init() {
7475
_ = imageregistryv1.AddToScheme(Scheme)
7576
_ = operatorsv1alpha1.AddToScheme(Scheme)
7677
_ = snapshotv1.AddToScheme(Scheme)
78+
_ = svmv1alpha1.AddToScheme(Scheme)
7779
}

‎control-plane-operator/hostedclusterconfigoperator/cmd.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/machine"
3030
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/node"
3131
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/nodecount"
32+
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/reencryption"
3233
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources"
3334
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/spotremediation"
3435
"github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/operator"
@@ -70,6 +71,7 @@ var controllerFuncs = map[string]operator.ControllerSetupFunc{
7071
"drainer": drainer.Setup,
7172
hcpstatus.ControllerName: hcpstatus.Setup,
7273
spotremediation.ControllerName: spotremediation.Setup,
74+
reencryption.ControllerName: reencryption.Setup,
7375
}
7476

7577
type HostedClusterConfigOperator struct {

0 commit comments

Comments
 (0)