Ingester: support zone-awareness

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Ingester: support zone-awareness #668
 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663
 
 ## 3.3.8 / 2026-08-17

templates/distributor/distributor-dep.yaml

@@ -54,6 +54,9 @@ spec:
           args:
             - "-target=distributor"
             - "-config.file=/etc/cortex/cortex.yaml"
+          {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }}
+            - "-distributor.zone-awareness-enabled"
+          {{- end }}
           {{- range $key, $value := .Values.distributor.extraArgs }}
             - "-{{ $key }}={{ $value }}"
           {{- end }}

templates/ingester/_helpers-ingester.tpl

@@ -21,3 +21,18 @@ ingester selector labels
 {{ include "cortex.selectorLabels" . }}
 app.kubernetes.io/component: ingester
 {{- end }}
+
+{{/*
+*/}}
+{{- define "cortex.ingesterZoneAwareReplicationMap" -}}
+{{- $zoneMap := dict }}
+{{- range $zone := .Values.ingester.zoneAwareReplication.zones }}
+{{- $_ := set $zone "stsSuffix" (printf "-%s" $zone.name) }}
+{{- $zoneMap := set $zoneMap $zone.name $zone }}
+{{- end }}
+{{- if or (not .Values.ingester.zoneAwareReplication.enabled) (and .Values.ingester.zoneAwareReplication.enabled .Values.ingester.zoneAwareReplication.migration) }}
+{{- $defaultZone := dict "stsSuffix" "" "name" "default" "nodeSelector" .Values.ingester.nodeSelector "replicas" .Values.ingester.replicas }}
+{{- $zoneMap := set $zoneMap "default" $defaultZone }}
+{{- end }}
+{{- toYaml $zoneMap }}
+{{- end }}

templates/ingester/ingester-statefulset.yaml

@@ -1,23 +1,38 @@
 {{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}}
+{{- $zoneMap := include "cortex.ingesterZoneAwareReplicationMap" $ | fromYaml }}
+{{- range $zoneName, $zone := $zoneMap }}
+{{- with $ }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: {{ include "cortex.ingesterFullname" . }}
+  name: {{ include "cortex.ingesterFullname" . }}{{ $zone.stsSuffix }}
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "cortex.ingesterLabels" . | nindent 4 }}
     app.kubernetes.io/part-of: memberlist
+    {{- with $zone.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with $zone.annotations }}
   annotations:
-    {{- toYaml .Values.ingester.annotations | nindent 4 }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   {{- if not .Values.ingester.autoscaling.enabled }}
-  replicas: {{ .Values.ingester.replicas }}
+  replicas: {{ $zone.replicas }}
   {{- end }}
   selector:
     matchLabels:
       {{- include "cortex.ingesterSelectorLabels" . | nindent 6 }}
+      {{- if .Values.ingester.zoneAwareReplication.enabled }}
+      zone: {{ $zoneName }}
+      {{- end }}
   updateStrategy:
+    {{- if ne $zoneName "default" }}
+    type: OnDelete
+    {{- else }}
     {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }}          
+    {{- end }}
   podManagementPolicy: "{{ .Values.ingester.statefulSet.podManagementPolicy }}"
   serviceName: {{ template "cortex.fullname" . }}-ingester-headless
   {{- if .Values.ingester.persistentVolume.enabled }}
@@ -34,6 +49,10 @@ spec:
         annotations:
           {{ toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }}
         {{- end }}
+        {{- with $zone.podLabels }}
+        labels:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         {{- if .Values.ingester.persistentVolume.storageClass }}
         {{- if (eq "-" .Values.ingester.persistentVolume.storageClass) }}
@@ -52,6 +71,9 @@ spec:
     metadata:
       labels:
         {{- include "cortex.ingesterLabels" . | nindent 8 }}
+        {{- if .Values.ingester.zoneAwareReplication.enabled }}
+        zone: {{ $zoneName }}
+        {{- end }}
         app.kubernetes.io/part-of: memberlist
         {{- with .Values.ingester.podLabels }}
         {{- toYaml . | nindent 8 }}
@@ -78,7 +100,7 @@ spec:
       {{- end }}
       {{- end }}
       nodeSelector:
-        {{- toYaml .Values.ingester.nodeSelector | nindent 8 }}
+        {{- toYaml $zone.nodeSelector | nindent 8 }}
       {{- if .Values.ingester.topologySpreadConstraints }}
       topologySpreadConstraints:
       {{- toYaml .Values.ingester.topologySpreadConstraints | nindent 8}}
@@ -110,6 +132,13 @@ spec:
           args:
             - "-target=ingester"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if .Values.ingester.zoneAwareReplication.enabled }}
+            - "-ingester.availability-zone={{ $zoneName }}"
+            {{- if .Values.ingester.zoneAwareReplication.migration }}
+            - "-blocks-storage.tsdb.flush-blocks-on-shutdown=true"
+            - "-ingester.unregister-on-shutdown=true"
+            {{- end }}
+            {{- end }}
             {{- include "cortex.memcached" . | nindent 12}}
             {{- range $key, $value := .Values.ingester.extraArgs }}
             - "-{{ $key }}={{ $value }}"
@@ -160,4 +189,7 @@ spec:
           lifecycle:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+---
+{{- end -}}
+{{- end -}}
 {{- end -}}

templates/querier/querier-dep.yaml

@@ -52,6 +52,9 @@ spec:
           args:
             - "-target=querier"
             - "-config.file=/etc/cortex/cortex.yaml"
+          {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }}
+            - "-distributor.zone-awareness-enabled"
+          {{- end }}
           {{- if .Values.query_scheduler.enabled }}
             - "-querier.scheduler-address={{ template "cortex.querySchedulerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.config.server.grpc_listen_port }}"
           {{- end }}

templates/ruler/ruler-dep.yaml

@@ -121,6 +121,9 @@ spec:
           args:
             - "-target=ruler"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if and .Values.ingester.zoneAwareReplication.enabled (not .Values.ingester.zoneAwareReplication.migration) }}
+            - "-distributor.zone-awareness-enabled"
+            {{- end }}
             {{- if and .Values.alertmanager.enabled (not .Values.config.ruler.alertmanager_url) }}
               {{- if .Values.config.ruler.enable_alertmanager_discovery }}
             - "-ruler.alertmanager-url=http://_http-metrics._tcp.{{ template "cortex.alertmanagerFullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}/api/prom/alertmanager/"

values.yaml

@@ -449,6 +449,23 @@ ingester:
     # -- ref: https://cortexmetrics.io/docs/guides/ingesters-scaling-up-and-down/#scaling-down and https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies for scaledown details
     podManagementPolicy: OrderedReady
 
+  zoneAwareReplication:
+    enabled: false
+    migration: false
+    zones: []
+    #  - name: eu02-1
+    #    nodeSelector:
+    #      topology.kubernetes.io/zone: eu02-1
+    #    labels: {}
+    #  - name: eu02-2
+    #    nodeSelector:
+    #      topology.kubernetes.io/zone: eu02-2
+    #    labels: {}
+    #  - name: eu02-3
+    #    nodeSelector:
+    #      topology.kubernetes.io/zone: eu02-3
+    #    labels: {}
+
   service:
     annotations: {}
     labels: {}
@@ -470,6 +487,8 @@ ingester:
   # -- Additional Cortex container arguments, e.g. log.level (debug, info, warn, error)
   extraArgs: {}
 
+  extraLabels: {}
+
   # -- Pod Labels
   podLabels: {}