diff --git a/CHANGELOG.md b/CHANGELOG.md index 010e7ca3..a46fc7d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [FEATURE] Ingester: support zone-awareness #668 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663 ## 3.3.8 / 2026-08-17 diff --git a/templates/distributor/distributor-dep.yaml b/templates/distributor/distributor-dep.yaml index 5470274e..75837036 100644 --- a/templates/distributor/distributor-dep.yaml +++ b/templates/distributor/distributor-dep.yaml @@ -54,6 +54,9 @@ spec: args: - "-target=distributor" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- range $key, $value := .Values.distributor.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/templates/ingester/_helpers-ingester.tpl b/templates/ingester/_helpers-ingester.tpl index 47053273..53975fef 100644 --- a/templates/ingester/_helpers-ingester.tpl +++ b/templates/ingester/_helpers-ingester.tpl @@ -21,3 +21,18 @@ ingester selector labels {{ include "cortex.selectorLabels" . }} app.kubernetes.io/component: ingester {{- end }} + +{{/* +*/}} +{{- define "cortex.ingesterZoneAwareReplicationMap" -}} +{{- $zoneMap := dict }} +{{- range $zone := .Values.ingester.zoneAwareReplication.zones }} +{{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }} +{{- $zoneMap := set $zoneMap $zone.name $zone }} +{{- end }} +{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration) }} +{{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }} +{{- $zoneMap := set $zoneMap "default" $defaultZone }} +{{- end }} +{{- toYaml $zoneMap }} +{{- end }} diff --git a/templates/ingester/ingester-statefulset.yaml b/templates/ingester/ingester-statefulset.yaml index b6d37ae2..9dcc561a 100644 --- a/templates/ingester/ingester-statefulset.yaml +++ b/templates/ingester/ingester-statefulset.yaml @@ -1,23 +1,38 @@ {{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}} +{{- $zoneMap := include "cortex.ingesterZoneAwareReplicationMap" $ | fromYaml }} +{{- range $zoneName, $zone := $zoneMap }} +{{- with $ }} apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "cortex.ingesterFullname" . }} + name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }} namespace: {{ .Release.Namespace }} labels: {{- include "cortex.ingesterLabels" . | nindent 4 }} app.kubernetes.io/part-of: memberlist + {{- with $zone.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $zone.annotations }} annotations: - {{- toYaml .Values.ingester.annotations | nindent 4 }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if not .Values.ingester.autoscaling.enabled }} - replicas: {{ .Values.ingester.replicas }} + replicas: {{ $zone.replicas }} {{- end }} selector: matchLabels: {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + zone: {{ $zoneName }} + {{- end }} updateStrategy: + {{- if ne $zoneName "default" }} + type: OnDelete + {{- else }} {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + {{- end }} podManagementPolicy: "{{ .Values.ingester.statefulSet.podManagementPolicy }}" serviceName: {{ template "cortex.fullname" . }}-ingester-headless {{- if .Values.ingester.persistentVolume.enabled }} @@ -34,6 +49,10 @@ spec: annotations: {{ toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }} {{- end }} + {{- with $zone.podLabels }} + labels: + {{- toYaml . | nindent 10 }} + {{- end }} spec: {{- if .Values.ingester.persistentVolume.storageClass }} {{- if (eq "-" .Values.ingester.persistentVolume.storageClass) }} @@ -52,6 +71,9 @@ spec: metadata: labels: {{- include "cortex.ingesterLabels" . | nindent 8 }} + {{- if .Values.ingester.zoneAwareReplication.enabled }} + zone: {{ $zoneName }} + {{- end }} app.kubernetes.io/part-of: memberlist {{- with .Values.ingester.podLabels }} {{- toYaml . | nindent 8 }} @@ -78,7 +100,7 @@ spec: {{- end }} {{- end }} nodeSelector: - {{- toYaml .Values.ingester.nodeSelector | nindent 8 }} + {{- toYaml $zone.nodeSelector | nindent 8 }} {{- if .Values.ingester.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml .Values.ingester.topologySpreadConstraints | nindent 8}} @@ -110,6 +132,13 @@ spec: args: - "-target=ingester" - "-config.file=/etc/cortex/cortex.yaml" + {{- if .Values.ingester.zoneAwareReplication.enabled }} + - "-ingester.availability-zone={{ $zoneName }}" + {{- if .Values.ingester.zoneAwareReplication.migration }} + - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true" + - "-ingester.unregister-on-shutdown=true" + {{- end }} + {{- end }} {{- include "cortex.memcached" . | nindent 12}} {{- range $key, $value := .Values.ingester.extraArgs }} - "-{{ $key }}={{ $value }}" @@ -160,4 +189,7 @@ spec: lifecycle: {{- toYaml . | nindent 12 }} {{- end }} +--- +{{- end -}} +{{- end -}} {{- end -}} diff --git a/templates/querier/querier-dep.yaml b/templates/querier/querier-dep.yaml index ee1f7493..138533c8 100644 --- a/templates/querier/querier-dep.yaml +++ b/templates/querier/querier-dep.yaml @@ -52,6 +52,9 @@ spec: args: - "-target=querier" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- if .Values.query_scheduler.enabled }} - "-querier.scheduler-address={{ template "cortex.querySchedulerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}" {{- end }} diff --git a/templates/ruler/ruler-dep.yaml b/templates/ruler/ruler-dep.yaml index 6e9f36fd..3316ae53 100644 --- a/templates/ruler/ruler-dep.yaml +++ b/templates/ruler/ruler-dep.yaml @@ -121,6 +121,9 @@ spec: args: - "-target=ruler" - "-config.file=/etc/cortex/cortex.yaml" + {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }} + - "-distributor.zone-awareness-enabled" + {{- end }} {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }} {{- if .Values.config.ruler.enable_alertmanager_discovery }} - "-ruler.alertmanager-url=http://_http-metrics._tcp.{{ template "cortex.alertmanagerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}/api/prom/alertmanager/" diff --git a/values.yaml b/values.yaml index 975a5469..dd6c1a61 100644 --- a/values.yaml +++ b/values.yaml @@ -449,6 +449,23 @@ ingester: # -- ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down and https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies for scaledown details podManagementPolicy: OrderedReady + zoneAwareReplication: + enabled: false + migration: false + zones: [] + # - name: eu02-1 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-1 + # labels: {} + # - name: eu02-2 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-2 + # labels: {} + # - name: eu02-3 + # nodeSelector: + # topology.kubernetes.io/zone: eu02-3 + # labels: {} + service: annotations: {} labels: {} @@ -470,6 +487,8 @@ ingester: # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error) extraArgs: {} + extraLabels: {} + # -- Pod Labels podLabels: {}