Skip to content

Commit 00fb137

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #3988 from nojnhuh/aks-cni-overlay
add spec.networkPluginMode to AzureManagedControlPlane
2 parents f261d6e + e43e9ea commit 00fb137

9 files changed

+230
-7
lines changed

‎api/v1beta1/azuremanagedcontrolplane_default.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ const (
3030
defaultAKSVnetCIDR = "10.0.0.0/8"
3131
// defaultAKSNodeSubnetCIDR is the default Node Subnet CIDR.
3232
defaultAKSNodeSubnetCIDR = "10.240.0.0/16"
33+
// defaultAKSVnetCIDRForOverlay is the default Vnet CIDR when Azure CNI overlay is enabled.
34+
defaultAKSVnetCIDRForOverlay = "10.224.0.0/12"
35+
// defaultAKSNodeSubnetCIDRForOverlay is the default Node Subnet CIDR when Azure CNI overlay is enabled.
36+
defaultAKSNodeSubnetCIDRForOverlay = "10.224.0.0/16"
3337
)
3438

3539
// setDefaultSSHPublicKey sets the default SSHPublicKey for an AzureManagedControlPlane.
@@ -60,6 +64,9 @@ func (m *AzureManagedControlPlane) setDefaultVirtualNetwork() {
6064
}
6165
if m.Spec.VirtualNetwork.CIDRBlock == "" {
6266
m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR
67+
if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay {
68+
m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDRForOverlay
69+
}
6370
}
6471
if m.Spec.VirtualNetwork.ResourceGroup == "" {
6572
m.Spec.VirtualNetwork.ResourceGroup = m.Spec.ResourceGroupName
@@ -73,6 +80,9 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() {
7380
}
7481
if m.Spec.VirtualNetwork.Subnet.CIDRBlock == "" {
7582
m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDR
83+
if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay {
84+
m.Spec.VirtualNetwork.Subnet.CIDRBlock = defaultAKSNodeSubnetCIDRForOverlay
85+
}
7686
}
7787
}
7888

‎api/v1beta1/azuremanagedcontrolplane_types.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,18 @@ const (
6262
ManagedControlPlaneIdentityTypeUserAssigned ManagedControlPlaneIdentityType = ManagedControlPlaneIdentityType(VMIdentityUserAssigned)
6363
)
6464

65+
// NetworkPluginMode is the mode the network plugin should use.
66+
type NetworkPluginMode string
67+
68+
const (
69+
// NetworkPluginModeOverlay is used with networkPlugin=azure, pods are given IPs from the PodCIDR address space but use Azure
70+
// Routing Domains rather than Kubenet's method of route tables.
71+
// See also [AKS doc].
72+
//
73+
// [AKS doc]: https://aka.ms/aks/azure-cni-overlay
74+
NetworkPluginModeOverlay NetworkPluginMode = "overlay"
75+
)
76+
6577
// AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane.
6678
type AzureManagedControlPlaneSpec struct {
6779
// Version defines the desired Kubernetes version.
@@ -110,6 +122,12 @@ type AzureManagedControlPlaneSpec struct {
110122
// +optional
111123
NetworkPlugin *string `json:"networkPlugin,omitempty"`
112124

125+
// NetworkPluginMode is the mode the network plugin should use.
126+
// Allowed value is "overlay".
127+
// +kubebuilder:validation:Enum=overlay
128+
// +optional
129+
NetworkPluginMode *NetworkPluginMode `json:"networkPluginMode,omitempty"`
130+
113131
// NetworkPolicy used for building Kubernetes network.
114132
// Allowed values are "azure", "calico".
115133
// Immutable.

‎api/v1beta1/azuremanagedcontrolplane_webhook.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o
258258
allErrs = append(allErrs, errs...)
259259
}
260260

261+
if errs := m.validateNetworkPluginModeUpdate(old); len(errs) > 0 {
262+
allErrs = append(allErrs, errs...)
263+
}
264+
261265
if errs := m.validateOIDCIssuerProfileUpdate(old); len(errs) > 0 {
262266
allErrs = append(allErrs, errs...)
263267
}
@@ -285,6 +289,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error {
285289
m.validateManagedClusterNetwork,
286290
m.validateAutoScalerProfile,
287291
m.validateIdentity,
292+
m.validateNetworkPluginMode,
288293
}
289294

290295
var errs []error
@@ -543,6 +548,17 @@ func (m *AzureManagedControlPlane) validateVirtualNetworkUpdate(old *AzureManage
543548
return allErrs
544549
}
545550

551+
// validateNetworkPluginModeUpdate validates update to NetworkPluginMode.
552+
func (m *AzureManagedControlPlane) validateNetworkPluginModeUpdate(old *AzureManagedControlPlane) field.ErrorList {
553+
var allErrs field.ErrorList
554+
555+
if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay && old.Spec.NetworkPolicy != nil {
556+
allErrs = append(allErrs, field.Forbidden(field.NewPath("Spec", "NetworkPluginMode"), fmt.Sprintf("%q NetworkPolicyMode cannot be enabled when NetworkPolicy is set", NetworkPluginModeOverlay)))
557+
}
558+
559+
return allErrs
560+
}
561+
546562
// validateOIDCIssuerProfile validates an OIDCIssuerProfile.
547563
func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureManagedControlPlane) field.ErrorList {
548564
var allErrs field.ErrorList
@@ -737,3 +753,20 @@ func (m *AzureManagedControlPlane) validateIdentity(_ client.Client) error {
737753

738754
return nil
739755
}
756+
757+
// validateNetworkPluginMode validates a NetworkPluginMode.
758+
func (m *AzureManagedControlPlane) validateNetworkPluginMode(_ client.Client) error {
759+
var allErrs field.ErrorList
760+
761+
const kubenet = "kubenet"
762+
if ptr.Deref(m.Spec.NetworkPluginMode, "") == NetworkPluginModeOverlay &&
763+
ptr.Deref(m.Spec.NetworkPlugin, "") == kubenet {
764+
allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "NetworkPluginMode"), m.Spec.NetworkPluginMode, fmt.Sprintf("cannot be set to %q when NetworkPlugin is %q", NetworkPluginModeOverlay, kubenet)))
765+
}
766+
767+
if len(allErrs) > 0 {
768+
return kerrors.NewAggregate(allErrs.ToAggregate().Errors())
769+
}
770+
771+
return nil
772+
}

‎api/v1beta1/azuremanagedcontrolplane_webhook_test.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,9 @@ func TestDefaultingWebhook(t *testing.T) {
5353
g.Expect(*amcp.Spec.SSHPublicKey).NotTo(BeEmpty())
5454
g.Expect(amcp.Spec.NodeResourceGroupName).To(Equal("MC_fooRg_fooName_fooLocation"))
5555
g.Expect(amcp.Spec.VirtualNetwork.Name).To(Equal("fooName"))
56+
g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDR))
5657
g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName"))
58+
g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDR))
5759
g.Expect(amcp.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier))
5860
g.Expect(amcp.Spec.Identity.Type).To(Equal(ManagedControlPlaneIdentityTypeSystemAssigned))
5961
g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeFalse())
@@ -87,6 +89,24 @@ func TestDefaultingWebhook(t *testing.T) {
8789
g.Expect(amcp.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooSubnetName"))
8890
g.Expect(amcp.Spec.SKU.Tier).To(Equal(PaidManagedControlPlaneTier))
8991
g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue())
92+
93+
t.Logf("Testing amcp defaulting webhook with overlay")
94+
amcp = &AzureManagedControlPlane{
95+
ObjectMeta: metav1.ObjectMeta{
96+
Name: "fooName",
97+
},
98+
Spec: AzureManagedControlPlaneSpec{
99+
ResourceGroupName: "fooRg",
100+
Location: "fooLocation",
101+
Version: "1.17.5",
102+
SSHPublicKey: ptr.To(""),
103+
NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
104+
},
105+
}
106+
err = mcpw.Default(context.Background(), amcp)
107+
g.Expect(err).NotTo(HaveOccurred())
108+
g.Expect(amcp.Spec.VirtualNetwork.CIDRBlock).To(Equal(defaultAKSVnetCIDRForOverlay))
109+
g.Expect(amcp.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDRForOverlay))
90110
}
91111

92112
func TestValidatingWebhook(t *testing.T) {
@@ -667,6 +687,30 @@ func TestValidatingWebhook(t *testing.T) {
667687
},
668688
expectErr: true,
669689
},
690+
{
691+
name: "overlay cannot be used with kubenet",
692+
amcp: AzureManagedControlPlane{
693+
ObjectMeta: getAMCPMetaData(),
694+
Spec: AzureManagedControlPlaneSpec{
695+
Version: "v1.24.1",
696+
NetworkPlugin: ptr.To("kubenet"),
697+
NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
698+
},
699+
},
700+
expectErr: true,
701+
},
702+
{
703+
name: "overlay can be used with azure",
704+
amcp: AzureManagedControlPlane{
705+
ObjectMeta: getAMCPMetaData(),
706+
Spec: AzureManagedControlPlaneSpec{
707+
Version: "v1.24.1",
708+
NetworkPlugin: ptr.To("azure"),
709+
NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
710+
},
711+
},
712+
expectErr: false,
713+
},
670714
}
671715

672716
for _, tt := range tests {
@@ -1422,6 +1466,49 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) {
14221466
},
14231467
wantErr: true,
14241468
},
1469+
{
1470+
name: "NetworkPluginMode cannot change to \"overlay\" when NetworkPolicy is set",
1471+
oldAMCP: &AzureManagedControlPlane{
1472+
ObjectMeta: metav1.ObjectMeta{
1473+
Name: "test-cluster",
1474+
},
1475+
Spec: AzureManagedControlPlaneSpec{
1476+
NetworkPolicy: ptr.To("anything"),
1477+
NetworkPluginMode: nil,
1478+
},
1479+
},
1480+
amcp: &AzureManagedControlPlane{
1481+
ObjectMeta: metav1.ObjectMeta{
1482+
Name: "test-cluster",
1483+
},
1484+
Spec: AzureManagedControlPlaneSpec{
1485+
NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
1486+
},
1487+
},
1488+
wantErr: true,
1489+
},
1490+
{
1491+
name: "NetworkPluginMode can change to \"overlay\" when NetworkPolicy is not set",
1492+
oldAMCP: &AzureManagedControlPlane{
1493+
ObjectMeta: metav1.ObjectMeta{
1494+
Name: "test-cluster",
1495+
},
1496+
Spec: AzureManagedControlPlaneSpec{
1497+
NetworkPolicy: nil,
1498+
NetworkPluginMode: nil,
1499+
},
1500+
},
1501+
amcp: &AzureManagedControlPlane{
1502+
ObjectMeta: metav1.ObjectMeta{
1503+
Name: "test-cluster",
1504+
},
1505+
Spec: AzureManagedControlPlaneSpec{
1506+
NetworkPluginMode: ptr.To(NetworkPluginModeOverlay),
1507+
Version: "v0.0.0",
1508+
},
1509+
},
1510+
wantErr: false,
1511+
},
14251512
{
14261513
name: "AzureManagedControlPlane OIDCIssuerProfile.Enabled false -> false OK",
14271514
oldAMCP: &AzureManagedControlPlane{

‎api/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎azure/scope/managedcontrolplane.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ResourceSpecGetter
484484
OutboundType: s.ControlPlane.Spec.OutboundType,
485485
Identity: s.ControlPlane.Spec.Identity,
486486
KubeletUserAssignedIdentity: s.ControlPlane.Spec.KubeletUserAssignedIdentity,
487+
NetworkPluginMode: s.ControlPlane.Spec.NetworkPluginMode,
487488
}
488489

489490
if s.ControlPlane.Spec.SSHPublicKey != nil {

‎azure/services/managedclusters/spec.go‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ type ManagedClusterSpec struct {
6666
// NetworkPlugin used for building Kubernetes network. Possible values include: 'azure', 'kubenet'. Defaults to azure.
6767
NetworkPlugin string
6868

69+
// NetworkPluginMode is the mode the network plugin should use.
70+
NetworkPluginMode *infrav1.NetworkPluginMode
71+
6972
// NetworkPolicy used for building Kubernetes network. Possible values include: 'calico', 'azure'.
7073
NetworkPolicy string
7174

@@ -351,6 +354,10 @@ func (s *ManagedClusterSpec) Parameters(ctx context.Context, existing interface{
351354
}
352355
}
353356

357+
if s.NetworkPluginMode != nil {
358+
managedCluster.Properties.NetworkProfile.NetworkPluginMode = ptr.To(armcontainerservice.NetworkPluginMode(*s.NetworkPluginMode))
359+
}
360+
354361
if s.PodCIDR != "" {
355362
managedCluster.Properties.NetworkProfile.PodCidr = &s.PodCIDR
356363
}
@@ -590,12 +597,18 @@ func computeDiffOfNormalizedClusters(managedCluster armcontainerservice.ManagedC
590597
}
591598
}
592599

600+
if existingMC.Properties.NetworkProfile != nil {
601+
existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile
602+
603+
existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode = existingMC.Properties.NetworkProfile.NetworkPluginMode
604+
}
593605
if managedCluster.Properties.NetworkProfile != nil {
594606
propertiesNormalized.NetworkProfile.LoadBalancerProfile = managedCluster.Properties.NetworkProfile.LoadBalancerProfile
595-
}
596607

597-
if existingMC.Properties.NetworkProfile != nil {
598-
existingMCPropertiesNormalized.NetworkProfile.LoadBalancerProfile = existingMC.Properties.NetworkProfile.LoadBalancerProfile
608+
propertiesNormalized.NetworkProfile.NetworkPluginMode = managedCluster.Properties.NetworkProfile.NetworkPluginMode
609+
if propertiesNormalized.NetworkProfile.NetworkPluginMode == nil {
610+
propertiesNormalized.NetworkProfile.NetworkPluginMode = existingMCPropertiesNormalized.NetworkProfile.NetworkPluginMode
611+
}
599612
}
600613

601614
if managedCluster.Properties.APIServerAccessProfile != nil {

‎azure/services/managedclusters/spec_test.go‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,10 @@ func TestParameters(t *testing.T) {
6868
Tags: map[string]string{
6969
"test-tag": "test-value",
7070
},
71-
Version: "v1.22.0",
72-
LoadBalancerSKU: "standard",
73-
SSHPublicKey: base64.StdEncoding.EncodeToString([]byte("test-ssh-key")),
71+
Version: "v1.22.0",
72+
LoadBalancerSKU: "standard",
73+
SSHPublicKey: base64.StdEncoding.EncodeToString([]byte("test-ssh-key")),
74+
NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay),
7475
OIDCIssuerProfile: &OIDCIssuerProfile{
7576
Enabled: ptr.To(true),
7677
},
@@ -415,6 +416,54 @@ func TestParameters(t *testing.T) {
415416
g.Expect(result).To(BeNil())
416417
},
417418
},
419+
{
420+
name: "setting networkPluginMode from nil to \"overlay\" will update",
421+
existing: func() armcontainerservice.ManagedCluster {
422+
c := getExistingCluster()
423+
c.Properties.NetworkProfile.NetworkPluginMode = nil
424+
return c
425+
}(),
426+
spec: &ManagedClusterSpec{
427+
Name: "test-managedcluster",
428+
ResourceGroup: "test-rg",
429+
Location: "test-location",
430+
Tags: map[string]string{
431+
"test-tag": "test-value",
432+
},
433+
Version: "v1.22.0",
434+
LoadBalancerSKU: "standard",
435+
OIDCIssuerProfile: &OIDCIssuerProfile{
436+
Enabled: ptr.To(true),
437+
},
438+
NetworkPluginMode: ptr.To(infrav1.NetworkPluginModeOverlay),
439+
},
440+
expect: func(g *WithT, result interface{}) {
441+
g.Expect(result).To(BeAssignableToTypeOf(armcontainerservice.ManagedCluster{}))
442+
g.Expect(result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).NotTo(BeNil())
443+
g.Expect(*result.(armcontainerservice.ManagedCluster).Properties.NetworkProfile.NetworkPluginMode).To(Equal(armcontainerservice.NetworkPluginModeOverlay))
444+
},
445+
},
446+
{
447+
name: "setting networkPluginMode from \"overlay\" to nil doesn't require update",
448+
existing: getExistingCluster(),
449+
spec: &ManagedClusterSpec{
450+
Name: "test-managedcluster",
451+
ResourceGroup: "test-rg",
452+
Location: "test-location",
453+
Tags: map[string]string{
454+
"test-tag": "test-value",
455+
},
456+
Version: "v1.22.0",
457+
LoadBalancerSKU: "standard",
458+
OIDCIssuerProfile: &OIDCIssuerProfile{
459+
Enabled: ptr.To(true),
460+
},
461+
NetworkPluginMode: nil,
462+
},
463+
expect: func(g *WithT, result interface{}) {
464+
g.Expect(result).To(BeNil())
465+
},
466+
},
418467
{
419468
name: "update needed when oidc issuer profile enabled changes",
420469
existing: getExistingCluster(),
@@ -584,7 +633,8 @@ func getSampleManagedCluster() armcontainerservice.ManagedCluster {
584633
NodeResourceGroup: ptr.To("test-node-rg"),
585634
EnableRBAC: ptr.To(true),
586635
NetworkProfile: &armcontainerservice.NetworkProfile{
587-
LoadBalancerSKU: ptr.To(armcontainerservice.LoadBalancerSKUStandard),
636+
LoadBalancerSKU: ptr.To(armcontainerservice.LoadBalancerSKUStandard),
637+
NetworkPluginMode: ptr.To(armcontainerservice.NetworkPluginModeOverlay),
588638
},
589639
OidcIssuerProfile: &armcontainerservice.ManagedClusterOIDCIssuerProfile{
590640
Enabled: ptr.To(true),

‎config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,12 @@ spec:
375375
- azure
376376
- kubenet
377377
type: string
378+
networkPluginMode:
379+
description: NetworkPluginMode is the mode the network plugin should
380+
use. Allowed value is "overlay".
381+
enum:
382+
- overlay
383+
type: string
378384
networkPolicy:
379385
description: NetworkPolicy used for building Kubernetes network. Allowed
380386
values are "azure", "calico". Immutable.

0 commit comments

Comments
 (0)