From d93850db1265e9ddaeb489249913776b00b3df88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 30 Apr 2024 21:05:33 +0200 Subject: [PATCH] Feature: nlb zone affinity via r53 (#694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feature: set nlb zone affinity for clients This enables you to set dns_record.client_routing_policy configuration, such that you can decide for NLB via R53 feature that clients running in the same zone will hit also the same zone of your NLB. This is interesting for cross cluster traffic or traffic that stays in the same cluster but passes the NLB. Valid configurations are: - availability_zone_affinity - partial_availability_zone_affinity - any_availability_zone (default) see also https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html\#load-balancer-attributes and https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html\#zonal-dns-affinity Signed-off-by: Sandor Szücs * log: log default cross zone and zone affinity config Signed-off-by: Sandor Szücs * fix: comments Signed-off-by: Sandor Szücs --------- Signed-off-by: Sandor Szücs --- aws/adapter.go | 16 +++++++ aws/cf.go | 1 + aws/cf_template.go | 14 ++++++- aws/cf_template_test.go | 42 +++++++++++++++++++ controller.go | 6 +++ controller_test.go | 1 + testdata/ingress_nlb/input/k8s/ing.yaml | 2 +- testdata/ingress_nlb/output/templates/ing.cf | 4 ++ .../output/templates/shared.cf | 4 ++ testdata/rg_nlb/output/templates/rg.cf | 4 ++ worker_test.go | 4 +- 11 files changed, 95 insertions(+), 3 deletions(-) diff --git a/aws/adapter.go b/aws/adapter.go index 19df8977..c14cfef6 100644 --- a/aws/adapter.go +++ b/aws/adapter.go @@ -66,6 +66,7 @@ type Adapter struct { ipAddressType string albLogsS3Bucket string albLogsS3Prefix string + nlbZoneAffinity string httpRedirectToHTTPS bool nlbCrossZone bool nlbHTTPEnabled bool @@ -124,8 +125,13 @@ const ( DefaultAlbS3LogsPrefix = "" DefaultCustomFilter = "" + + // DefaultZoneAffinity specifies dns_record.client_routing_policy, see also https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes + DefaultZoneAffinity = "any_availability_zone" + // DefaultNLBCrossZone specifies the default configuration for cross // zone load balancing: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes + // It it is safe to change as per https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticloadbalancingv2-loadbalancer-loadbalancerattribute.html#aws-properties-elasticloadbalancingv2-loadbalancer-loadbalancerattribute-properties DefaultNLBCrossZone = false DefaultNLBHTTPEnabled = false @@ -246,6 +252,7 @@ func NewAdapter(clusterID, newControllerID, vpcID string, debug, disableInstrume albLogsS3Bucket: DefaultAlbS3LogsBucket, albLogsS3Prefix: DefaultAlbS3LogsPrefix, nlbCrossZone: DefaultNLBCrossZone, + nlbZoneAffinity: DefaultZoneAffinity, nlbHTTPEnabled: DefaultNLBHTTPEnabled, customFilter: DefaultCustomFilter, TargetCNI: &TargetCNIconfig{ @@ -446,6 +453,13 @@ func (a *Adapter) WithNLBCrossZone(nlbCrossZone bool) *Adapter { return a } +// WithNLBZoneAffinity returns the receiver adapter after setting the +// nlbZoneAffinity config. +func (a *Adapter) WithNLBZoneAffinity(nlbZoneAffinity string) *Adapter { + a.nlbZoneAffinity = nlbZoneAffinity + return a +} + // WithNLBHTTPEnabled returns the receiver adapter after setting the // nlbHTTPEnabled config. func (a *Adapter) WithNLBHTTPEnabled(nlbHTTPEnabled bool) *Adapter { @@ -753,6 +767,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o cwAlarms: cwAlarms, httpRedirectToHTTPS: a.httpRedirectToHTTPS, nlbCrossZone: a.nlbCrossZone, + nlbZoneAffinity: a.nlbZoneAffinity, http2: http2, tags: a.stackTags, internalDomains: a.internalDomains, @@ -809,6 +824,7 @@ func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time. cwAlarms: cwAlarms, httpRedirectToHTTPS: a.httpRedirectToHTTPS, nlbCrossZone: a.nlbCrossZone, + nlbZoneAffinity: a.nlbZoneAffinity, http2: http2, tags: a.stackTags, internalDomains: a.internalDomains, diff --git a/aws/cf.go b/aws/cf.go index c9ed72a7..1a4ab47a 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -179,6 +179,7 @@ type stackSpec struct { albLogsS3Bucket string albLogsS3Prefix string wafWebAclId string + nlbZoneAffinity string cwAlarms CloudWatchAlarmList httpRedirectToHTTPS bool nlbCrossZone bool diff --git a/aws/cf_template.go b/aws/cf_template.go index 2e3e0654..145cb805 100644 --- a/aws/cf_template.go +++ b/aws/cf_template.go @@ -282,7 +282,7 @@ func generateTemplate(spec *stackSpec) (string, error) { } // Build up the LoadBalancerAttributes list, as there is no way to make attributes conditional in the template - lbAttrList := make(cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttributeList, 0, 4) + lbAttrList := make(cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttributeList, 0, 5) if spec.loadbalancerType == LoadBalancerTypeApplication { lbAttrList = append(lbAttrList, @@ -307,6 +307,18 @@ func generateTemplate(spec *stackSpec) (string, error) { Value: cloudformation.String(fmt.Sprintf("%t", spec.nlbCrossZone)), }, ) + + lbAttrList = append(lbAttrList, + cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute{ + // https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes + // https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#zonal-dns-affinity + Key: cloudformation.String("dns_record.client_routing_policy"), + // availability_zone_affinity 100% + // partial_availability_zone_affinity 85% + // any_availability_zone 0% + Value: cloudformation.String(spec.nlbZoneAffinity), + }, + ) } if spec.albLogsS3Bucket != "" { diff --git a/aws/cf_template_test.go b/aws/cf_template_test.go index 103684d2..49c93b97 100644 --- a/aws/cf_template_test.go +++ b/aws/cf_template_test.go @@ -291,6 +291,48 @@ func TestGenerateTemplate(t *testing.T) { require.Equal(t, attributes[0].Value.Literal, "false") }, }, + { + name: "nlb zone affinity Route53 can be configured to any_availability_zone for Network load balancers", + spec: &stackSpec{ + loadbalancerType: LoadBalancerTypeNetwork, + nlbZoneAffinity: "any_availability_zone", + }, + validate: func(t *testing.T, template *cloudformation.Template) { + require.NotNil(t, template.Resources["LB"]) + properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer) + attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) + require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy") + require.Equal(t, attributes[1].Value.Literal, "any_availability_zone") + }, + }, + { + name: "nlb zone affinity Route53 can be configured to availability_zone_affinity for Network load balancers", + spec: &stackSpec{ + loadbalancerType: LoadBalancerTypeNetwork, + nlbZoneAffinity: "availability_zone_affinity", + }, + validate: func(t *testing.T, template *cloudformation.Template) { + require.NotNil(t, template.Resources["LB"]) + properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer) + attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) + require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy") + require.Equal(t, attributes[1].Value.Literal, "availability_zone_affinity") + }, + }, + { + name: "nlb zone affinity Route53 can be configured to partial_availability_zone_affinity for Network load balancers", + spec: &stackSpec{ + loadbalancerType: LoadBalancerTypeNetwork, + nlbZoneAffinity: "partial_availability_zone_affinity", + }, + validate: func(t *testing.T, template *cloudformation.Template) { + require.NotNil(t, template.Resources["LB"]) + properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer) + attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) + require.Equal(t, attributes[1].Key.Literal, "dns_record.client_routing_policy") + require.Equal(t, attributes[1].Value.Literal, "partial_availability_zone_affinity") + }, + }, { name: "nlb HTTP listener should not be enabled when HTTP is disabled", spec: &stackSpec{ diff --git a/controller.go b/controller.go index 76e0ee1c..a0d66b7d 100644 --- a/controller.go +++ b/controller.go @@ -76,6 +76,7 @@ var ( cwAlarmConfigMap string cwAlarmConfigMapLocation *kubernetes.ResourceLocation loadBalancerType string + nlbZoneAffinity string nlbCrossZone bool nlbHTTPEnabled bool ingressAPIVersion string @@ -175,6 +176,8 @@ func loadSettings() error { Default(defaultHTTPRedirectToHTTPS).BoolVar(&httpRedirectToHTTPS) kingpin.Flag("load-balancer-type", "Sets default Load Balancer type (application or network)."). Default(aws.LoadBalancerTypeApplication).EnumVar(&loadBalancerType, aws.LoadBalancerTypeApplication, aws.LoadBalancerTypeNetwork) + kingpin.Flag("nlb-zone-affinity", "Specify whether Route53 should return zone aware Network Load Balancers IPs. It configures dns_record.client_routing_policy NLB configuration. This setting only apply to 'network' Load Balancers."). + Default(aws.DefaultZoneAffinity).StringVar(&nlbZoneAffinity) kingpin.Flag("nlb-cross-zone", "Specify whether Network Load Balancers should balance cross availablity zones. This setting only apply to 'network' Load Balancers."). Default("false").BoolVar(&nlbCrossZone) kingpin.Flag("nlb-http-enabled", "Enable HTTP (port 80) for Network Load Balancers. By default this is disabled as NLB can't provide HTTP -> HTTPS redirect."). @@ -339,6 +342,7 @@ func main() { WithAlbLogsS3Prefix(albLogsS3Prefix). WithHTTPRedirectToHTTPS(httpRedirectToHTTPS). WithNLBCrossZone(nlbCrossZone). + WithNLBZoneAffinity(nlbZoneAffinity). WithNLBHTTPEnabled(nlbHTTPEnabled). WithCustomFilter(customFilter). WithStackTags(additionalStackTags). @@ -409,6 +413,8 @@ func main() { log.Infof("CloudWatch Alarm ConfigMap: %s", cwAlarmConfigMapLocation) log.Infof("Default LoadBalancer type: %s", loadBalancerType) log.Infof("Target access mode: %s", targetAccessMode) + log.Infof("NLB Cross Zone: %t", nlbCrossZone) + log.Infof("NLB Zone Affinity: %s", nlbZoneAffinity) go handleTerminationSignals(cancel, syscall.SIGTERM, syscall.SIGQUIT) go serveMetrics(metricsAddress) diff --git a/controller_test.go b/controller_test.go index 30ada2fa..18857ec7 100644 --- a/controller_test.go +++ b/controller_test.go @@ -67,6 +67,7 @@ func TestDefaultLoadSettings(t *testing.T) { require.Equal(t, "", cwAlarmConfigMap) require.Equal(t, (*kubernetes.ResourceLocation)(nil), cwAlarmConfigMapLocation) require.Equal(t, "application", loadBalancerType) + require.Equal(t, "any_availability_zone", nlbZoneAffinity) require.Equal(t, false, nlbCrossZone) require.Equal(t, false, nlbHTTPEnabled) require.Equal(t, "networking.k8s.io/v1", ingressAPIVersion) diff --git a/testdata/ingress_nlb/input/k8s/ing.yaml b/testdata/ingress_nlb/input/k8s/ing.yaml index 1f21110d..f13da3da 100644 --- a/testdata/ingress_nlb/input/k8s/ing.yaml +++ b/testdata/ingress_nlb/input/k8s/ing.yaml @@ -3,7 +3,7 @@ kind: Ingress metadata: name: myingress annotations: - zalando.org/aws-load-balancer-type: nlb + zalando.org/aws-load-balancer-type: nlb spec: rules: - host: foo.bar.org diff --git a/testdata/ingress_nlb/output/templates/ing.cf b/testdata/ingress_nlb/output/templates/ing.cf index 984e9cc0..d024df80 100644 --- a/testdata/ingress_nlb/output/templates/ing.cf +++ b/testdata/ingress_nlb/output/templates/ing.cf @@ -116,6 +116,10 @@ "Key": "load_balancing.cross_zone.enabled", "Value": "false" }, + { + "Key": "dns_record.client_routing_policy", + "Value": "any_availability_zone" + }, { "Key": "access_logs.s3.enabled", "Value": "false" diff --git a/testdata/ingress_rg_shared_nlb/output/templates/shared.cf b/testdata/ingress_rg_shared_nlb/output/templates/shared.cf index 984e9cc0..d024df80 100644 --- a/testdata/ingress_rg_shared_nlb/output/templates/shared.cf +++ b/testdata/ingress_rg_shared_nlb/output/templates/shared.cf @@ -116,6 +116,10 @@ "Key": "load_balancing.cross_zone.enabled", "Value": "false" }, + { + "Key": "dns_record.client_routing_policy", + "Value": "any_availability_zone" + }, { "Key": "access_logs.s3.enabled", "Value": "false" diff --git a/testdata/rg_nlb/output/templates/rg.cf b/testdata/rg_nlb/output/templates/rg.cf index 984e9cc0..d024df80 100644 --- a/testdata/rg_nlb/output/templates/rg.cf +++ b/testdata/rg_nlb/output/templates/rg.cf @@ -116,6 +116,10 @@ "Key": "load_balancing.cross_zone.enabled", "Value": "false" }, + { + "Key": "dns_record.client_routing_policy", + "Value": "any_availability_zone" + }, { "Key": "access_logs.s3.enabled", "Value": "false" diff --git a/worker_test.go b/worker_test.go index bb0ff981..7c5ec51a 100644 --- a/worker_test.go +++ b/worker_test.go @@ -18,6 +18,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/zalando-incubator/kube-ingress-aws-controller/aws" awsAdapter "github.com/zalando-incubator/kube-ingress-aws-controller/aws" "github.com/zalando-incubator/kube-ingress-aws-controller/certs" "github.com/zalando-incubator/kube-ingress-aws-controller/kubernetes" @@ -471,7 +472,8 @@ func TestResourceConversionOneToOne(tt *testing.T) { a = a.WithCustomAutoScalingClient(clientASG). WithCustomEc2Client(clientEC2). WithCustomElbv2Client(clientELBv2). - WithCustomCloudFormationClient(clientCF) + WithCustomCloudFormationClient(clientCF). + WithNLBZoneAffinity(aws.DefaultZoneAffinity) a, err = a.UpdateManifest(clusterID, vpcID) if err != nil {