Skip to content

Commit 96575b7

Browse files
bertinattobertinatto
committed
wip
1 parent f319426 commit 96575b7

3 files changed

Lines changed: 207 additions & 27 deletions

File tree

‎pkg/operator/targetconfigcontroller/kms_sidecar.go‎

Lines changed: 31 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ package targetconfigcontroller
33
import (
44
"encoding/json"
55
"fmt"
6+
"regexp"
67
"slices"
78
"strconv"
8-
"strings"
99

1010
"github.com/openshift/api/features"
1111
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
@@ -22,6 +22,7 @@ import (
2222
var (
2323
apiserverScheme = runtime.NewScheme()
2424
apiserverCodecs = serializer.NewCodecFactory(apiserverScheme)
25+
kmsNameRegex = regexp.MustCompile(`-(\d+)_`)
2526
)
2627

2728
func init() {
@@ -103,37 +104,47 @@ func AddKMSPluginToPodSpec(podSpec *corev1.PodSpec, featureGateAccessor featureg
103104
// klog.Infof("kms is disabled: failed to get vault-kms-credentials secret: %v", err)
104105
// return nil
105106
// }
106-
keyID := -1
107-
for k := range encryptionConfig.Data {
108-
if !strings.Contains(k, "kms-provider-config-") {
109-
continue
110-
}
111-
parts := strings.Split(k, "-")
112-
lastField := parts[len(parts)-1]
113-
id, err := strconv.Atoi(lastField)
114-
if err != nil {
115-
return err
107+
108+
// TODO: only the first KMS provider is used for now
109+
var kmsConfig *apiserverv1.KMSConfiguration
110+
for _, resource := range config.Resources {
111+
for _, provider := range resource.Providers {
112+
if provider.KMS != nil {
113+
kmsConfig = provider.KMS
114+
break
115+
}
116116
}
117-
// TODO: should never happen, right?
118-
if id > keyID {
119-
keyID = id
117+
if kmsConfig != nil {
118+
break
120119
}
121120
}
122121

123-
if keyID < 0 {
124-
return fmt.Errorf("no kms-provider-config-* key found in encryption config secret")
122+
if kmsConfig == nil || kmsConfig.Name == "" {
123+
return fmt.Errorf("no KMS provider found in EncryptionConfiguration")
124+
}
125+
126+
// name format: kms-<ID>_<resource>, e.g. "kms-2_secrets"
127+
kmsName := kmsConfig.Name
128+
m := kmsNameRegex.FindStringSubmatch(kmsName)
129+
if m == nil {
130+
return fmt.Errorf("unexpected KMS provider name format: %s", kmsName)
131+
}
132+
keyID, err := strconv.Atoi(m[1])
133+
if err != nil {
134+
return fmt.Errorf("failed to parse key ID from KMS provider name %q: %w", kmsName, err)
125135
}
126136

127137
keyKMSProviderConfig := fmt.Sprintf("kms-provider-config-%d", keyID)
128138
keySecretID := fmt.Sprintf("kms-secret-id-%d", keyID)
139+
endpoint := kmsConfig.Endpoint
129140

130141
vaultConfig := &vaultConfiguration{}
131142
if err := json.Unmarshal(encryptionConfig.Data[keyKMSProviderConfig], vaultConfig); err != nil {
132143
return err
133144
}
134145

135146
klog.Infof("kms is enabled: found config, now patching kube-apiserver pod")
136-
if err := addKMSPluginSidecarToPodSpec(podSpec, "kms-plugin", kmsPluginImage, vaultConfig, keySecretID); err != nil {
147+
if err := addKMSPluginSidecarToPodSpec(podSpec, "kms-plugin", kmsPluginImage, vaultConfig, endpoint, keySecretID); err != nil {
137148
return err
138149
}
139150

@@ -155,7 +166,7 @@ type vaultConfiguration struct {
155166
KeyName string
156167
}
157168

158-
func addKMSPluginSidecarToPodSpec(podSpec *corev1.PodSpec, containerName string, image string, config *vaultConfiguration, keySecretID string) error {
169+
func addKMSPluginSidecarToPodSpec(podSpec *corev1.PodSpec, containerName string, image string, config *vaultConfiguration, endpoint, keySecretID string) error {
159170
if podSpec == nil {
160171
return fmt.Errorf("pod spec cannot be nil")
161172
}
@@ -167,14 +178,15 @@ func addKMSPluginSidecarToPodSpec(podSpec *corev1.PodSpec, containerName string,
167178
Command: []string{"/bin/sh", "-c"},
168179
Args: []string{fmt.Sprintf(`
169180
exec /vault-kube-kms \
170-
-listen-address=unix:///var/run/kmsplugin/kms.sock \
181+
-listen-address=%s \
171182
-vault-address=%s \
172183
-vault-namespace=%s \
173184
-transit-mount=transit \
174185
-transit-key=%s \
175186
-log-level=debug-extended \
176187
-approle-role-id=%s \
177188
-approle-secret-id-path=/etc/kubernetes/static-pod-resources/%s`,
189+
endpoint,
178190
config.Addr,
179191
config.Namespace,
180192
config.KeyName,

‎pkg/operator/targetconfigcontroller/kms_sidecar_test.go‎

Lines changed: 175 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func TestAddKMSPluginToPodSpec(t *testing.T) {
4040
Addr: "https://vault.example.com:8200",
4141
Namespace: "my-namespace",
4242
KeyName: "my-key",
43-
}, "kms-secret-id-555"),
43+
}, "unix:///var/run/kmsplugin/kms-555.sock", "kms-secret-id-555"),
4444
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
4545
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
4646
nil,
@@ -55,6 +55,140 @@ func TestAddKMSPluginToPodSpec(t *testing.T) {
5555
},
5656
kmsPluginImage: "quay.io/example/vault-kms:v1",
5757
},
58+
{
59+
name: "KMS provider name with extra dashes",
60+
podSpec: &corev1.PodSpec{
61+
Containers: []corev1.Container{
62+
{Name: "kube-apiserver"},
63+
},
64+
},
65+
expectedPodSpec: expectedPodSpecWithKMSSidecar("quay.io/example/vault-kms:v1", &vaultConfiguration{
66+
RoleID: "test-role-id",
67+
Addr: "https://vault.example.com:8200",
68+
Namespace: "my-namespace",
69+
KeyName: "my-key",
70+
}, "unix:///var/run/kmsplugin/kms-3.sock", "kms-secret-id-3"),
71+
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
72+
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
73+
nil,
74+
),
75+
secrets: []*corev1.Secret{
76+
newEncryptionConfigSecretWithKMSName(t, "vault-kms-3_secrets", "unix:///var/run/kmsplugin/kms-3.sock", &vaultConfiguration{
77+
RoleID: "test-role-id",
78+
Addr: "https://vault.example.com:8200",
79+
Namespace: "my-namespace",
80+
KeyName: "my-key",
81+
}, 3),
82+
},
83+
kmsPluginImage: "quay.io/example/vault-kms:v1",
84+
},
85+
{
86+
name: "feature gate disabled: pod spec unchanged",
87+
podSpec: &corev1.PodSpec{
88+
Containers: []corev1.Container{
89+
{Name: "kube-apiserver"},
90+
},
91+
},
92+
expectedPodSpec: &corev1.PodSpec{
93+
Containers: []corev1.Container{
94+
{Name: "kube-apiserver"},
95+
},
96+
},
97+
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
98+
nil,
99+
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
100+
),
101+
},
102+
{
103+
name: "encryption config secret not found: pod spec unchanged",
104+
podSpec: &corev1.PodSpec{
105+
Containers: []corev1.Container{
106+
{Name: "kube-apiserver"},
107+
},
108+
},
109+
expectedPodSpec: &corev1.PodSpec{
110+
Containers: []corev1.Container{
111+
{Name: "kube-apiserver"},
112+
},
113+
},
114+
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
115+
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
116+
nil,
117+
),
118+
secrets: []*corev1.Secret{},
119+
},
120+
{
121+
name: "no KMS provider in EncryptionConfiguration: pod spec unchanged",
122+
podSpec: &corev1.PodSpec{
123+
Containers: []corev1.Container{
124+
{Name: "kube-apiserver"},
125+
},
126+
},
127+
expectedPodSpec: &corev1.PodSpec{
128+
Containers: []corev1.Container{
129+
{Name: "kube-apiserver"},
130+
},
131+
},
132+
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
133+
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
134+
nil,
135+
),
136+
secrets: []*corev1.Secret{
137+
{
138+
ObjectMeta: metav1.ObjectMeta{
139+
Name: "encryption-config-openshift-kube-apiserver",
140+
Namespace: "openshift-config-managed",
141+
},
142+
Data: map[string][]byte{
143+
"encryption-config": []byte(`
144+
apiVersion: apiserver.config.k8s.io/v1
145+
kind: EncryptionConfiguration
146+
resources:
147+
- resources:
148+
- secrets
149+
providers:
150+
- identity: {}
151+
`),
152+
},
153+
},
154+
},
155+
},
156+
{
157+
name: "malformed KMS provider name: error",
158+
podSpec: &corev1.PodSpec{
159+
Containers: []corev1.Container{
160+
{Name: "kube-apiserver"},
161+
},
162+
},
163+
featureGateAccessor: featuregates.NewHardcodedFeatureGateAccess(
164+
[]configv1.FeatureGateName{features.FeatureGateKMSEncryption},
165+
nil,
166+
),
167+
secrets: []*corev1.Secret{
168+
{
169+
ObjectMeta: metav1.ObjectMeta{
170+
Name: "encryption-config-openshift-kube-apiserver",
171+
Namespace: "openshift-config-managed",
172+
},
173+
Data: map[string][]byte{
174+
"encryption-config": []byte(`
175+
apiVersion: apiserver.config.k8s.io/v1
176+
kind: EncryptionConfiguration
177+
resources:
178+
- resources:
179+
- secrets
180+
providers:
181+
- kms:
182+
apiVersion: v2
183+
name: invalid-name
184+
endpoint: unix:///var/run/kmsplugin/kms.sock
185+
timeout: 10s
186+
`),
187+
},
188+
},
189+
},
190+
wantErr: "unexpected KMS provider name format",
191+
},
58192
}
59193

60194
for _, tt := range tests {
@@ -78,7 +212,7 @@ func TestAddKMSPluginToPodSpec(t *testing.T) {
78212
}
79213
}
80214

81-
func expectedPodSpecWithKMSSidecar(image string, config *vaultConfiguration, keySecretID string) *corev1.PodSpec {
215+
func expectedPodSpecWithKMSSidecar(image string, config *vaultConfiguration, endpoint, keySecretID string) *corev1.PodSpec {
82216
directoryOrCreate := corev1.HostPathDirectoryOrCreate
83217
return &corev1.PodSpec{
84218
Containers: []corev1.Container{
@@ -98,14 +232,15 @@ func expectedPodSpecWithKMSSidecar(image string, config *vaultConfiguration, key
98232
Command: []string{"/bin/sh", "-c"},
99233
Args: []string{fmt.Sprintf(`
100234
exec /vault-kube-kms \
101-
-listen-address=unix:///var/run/kmsplugin/kms.sock \
235+
-listen-address=%s \
102236
-vault-address=%s \
103237
-vault-namespace=%s \
104238
-transit-mount=transit \
105239
-transit-key=%s \
106240
-log-level=debug-extended \
107241
-approle-role-id=%s \
108242
-approle-secret-id-path=/etc/kubernetes/static-pod-resources/%s`,
243+
endpoint,
109244
config.Addr,
110245
config.Namespace,
111246
config.KeyName,
@@ -155,10 +290,10 @@ resources:
155290
- secrets
156291
providers:
157292
- kms:
158-
name: vault
159-
endpoint: unix:///var/run/kmsplugin/kms.sock
160-
cachesize: 1000
161-
timeout: 3s
293+
apiVersion: v2
294+
name: kms-555_secrets
295+
endpoint: unix:///var/run/kmsplugin/kms-555.sock
296+
timeout: 10s
162297
- identity: {}
163298
`
164299
providerConfig, err := json.Marshal(config)
@@ -177,6 +312,39 @@ resources:
177312
}
178313
}
179314

315+
func newEncryptionConfigSecretWithKMSName(t *testing.T, kmsName, endpoint string, config *vaultConfiguration, keyID int) *corev1.Secret {
316+
t.Helper()
317+
318+
encryptionConfig := fmt.Sprintf(`
319+
apiVersion: apiserver.config.k8s.io/v1
320+
kind: EncryptionConfiguration
321+
resources:
322+
- resources:
323+
- secrets
324+
providers:
325+
- kms:
326+
apiVersion: v2
327+
name: %s
328+
endpoint: %s
329+
timeout: 10s
330+
- identity: {}
331+
`, kmsName, endpoint)
332+
providerConfig, err := json.Marshal(config)
333+
require.NoError(t, err)
334+
335+
return &corev1.Secret{
336+
ObjectMeta: metav1.ObjectMeta{
337+
Name: "encryption-config-openshift-kube-apiserver",
338+
Namespace: "openshift-config-managed",
339+
},
340+
Data: map[string][]byte{
341+
"encryption-config": []byte(encryptionConfig),
342+
fmt.Sprintf("kms-provider-config-%d", keyID): providerConfig,
343+
fmt.Sprintf("kms-secret-id-%d", keyID): []byte("some-secret-id"),
344+
},
345+
}
346+
}
347+
180348
// secretLister implements corev1listers.SecretLister backed by a fake client.
181349
type secretLister struct {
182350
client *fake.Clientset

‎pkg/operator/targetconfigcontroller/targetconfigcontroller.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, featu
331331
}
332332

333333
// TODO: placeholder. I (fbertina) need to grant you read permissions. Please ask
334-
kmsPluginImage := "quay.io/bertinatto/vault:v1"
334+
kmsPluginImage := "quay.io/bertinatto/vault:v2"
335335

336336
// encryptionConfig, err := secretLister.Secrets(namespace string)
337337

0 commit comments

Comments
 (0)