Add zone awareness support for ingester

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Support zone awareness on Ingester. #632
 * [DEPENDENCY] update kiwigrid/k8s-sidecar docker tag to v2.11.2 #663
 
 ## 3.3.8 / 2026-08-17

README.md

@@ -382,6 +382,9 @@ Kubernetes: `^1.19.0-0`
 | ingester.​terminationGracePeriodSeconds | int | `240` |  |
 | ingester.​tolerations | list | `[]` |  |
 | ingester.​topologySpreadConstraints | list | `[]` |  |
+| ingester.&ZeroWidthSpace;zoneAwareness.&ZeroWidthSpace;enabled | bool | `false` | If true, ingesters are deployed as one StatefulSet (or Deployment) per entry in `zones`, each started with `-ingester.availability-zone=<zone name>`, and ingesters, distributors, queriers and rulers get `-distributor.zone-awareness-enabled`. Replicas of a series are spread across zones, so run at least as many zones as the replication factor to survive a zone outage, and the same number of replicas in every zone to keep the load balanced. |
+| ingester.&ZeroWidthSpace;zoneAwareness.&ZeroWidthSpace;migration.&ZeroWidthSpace;enabled | bool | `false` | If true, the zone-less ingester workload is kept alongside the zone workloads and `-distributor.zone-awareness-enabled` is set to false on every component. |
+| ingester.&ZeroWidthSpace;zoneAwareness.&ZeroWidthSpace;zones | list | `[{"name":"zone-a","nodeSelector":{},"replicas":1},{"name":"zone-b","nodeSelector":{},"replicas":1},{"name":"zone-c","nodeSelector":{},"replicas":1}]` | Zones to deploy ingesters into. `name` is required. `replicas`, `nodeSelector`, `affinity`, `tolerations` and `annotations` fall back to the top-level ingester values when unset; `podLabels` and `podAnnotations` are merged into them. |
 | ingress.&ZeroWidthSpace;annotations | object | `{}` |  |
 | ingress.&ZeroWidthSpace;enabled | bool | `false` |  |
 | ingress.&ZeroWidthSpace;hosts[0].&ZeroWidthSpace;host | string | `"chart-example.local"` |  |

ci/test-zone-aware-values.yaml

@@ -0,0 +1,188 @@
+ingress:
+  enabled: true
+
+config:
+  ingester:
+    lifecycler:
+      join_after: 30s
+      final_sleep: 30s
+      tokens_file_path: "/data/tokens"
+      ring:
+        replication_factor: 1
+        kvstore:
+          store: "memberlist"
+
+  blocks_storage:
+    backend: "filesystem"
+    tsdb:
+      dir: "/data/tsdb"
+    bucket_store:
+      sync_dir: "/data/tsdb-sync"
+    filesystem:
+      dir: "/data/store"
+
+  distributor:
+    ring:
+      kvstore:
+        store: memberlist
+
+  ruler:
+    ring:
+      kvstore:
+        store: memberlist
+
+  ruler_storage:
+    backend: "filesystem"
+    filesystem:
+      dir: "/data/store"
+
+  alertmanager_storage:
+    backend: "filesystem"
+    filesystem:
+      dir: "/data/store"
+
+  memberlist:
+    join_members:
+      - '{{ include "cortex.fullname" $ }}-memberlist'
+
+ingester:
+  replicas: 1
+  statefulSet:
+    enabled: true
+  zoneAwareness:
+    enabled: true
+    zones:
+      - name: zone-a
+        replicas: 1
+      - name: zone-b
+        replicas: 1
+      - name: zone-c
+        replicas: 1
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+compactor:
+  replicas: 1
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+distributor:
+  replicas: 1
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+querier:
+  replicas: 1
+  autoscaling:
+    enabled: true
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+query_frontend:
+  replicas: 1
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+nginx:
+  replicas: 1
+  autoscaling:
+    enabled: true
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+  config:
+    httpSnippet: |-
+      # http snippet
+    mainSnippet: |-
+      # main snippet
+    serverSnippet: |-
+      # server snippet
+runtimeconfigmap:
+  annotations:
+    foo: bar
+alertmanager:
+  replicas: 3
+  statefulSet:
+    enabled: true
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+  sidecar:
+    enabled: true
+ruler:
+  replicas: 1
+  enabled: true
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+  sidecar:
+    enabled: true
+store_gateway:
+  startupProbe: null
+  replicas: 1
+  extraVolumes:
+    - name: tmp-test
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: tmp-test
+      mountPath: /tmp-test
+
+memcached-frontend:
+  enabled: true
+  architecture: standalone
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: 0
+      memory: 0
+memcached-blocks:
+  enabled: true
+  architecture: standalone
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: 0
+      memory: 0
+memcached-blocks-index:
+  enabled: true
+  architecture: standalone
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: 0
+      memory: 0
+memcached-blocks-metadata:
+  enabled: true
+  architecture: standalone
+  replicaCount: 1
+  resources:
+    requests:
+      cpu: 0
+      memory: 0

templates/distributor/distributor-dep.yaml

@@ -54,6 +54,9 @@ spec:
           args:
             - "-target=distributor"
             - "-config.file=/etc/cortex/cortex.yaml"
+          {{- if .Values.ingester.zoneAwareness.enabled }}
+            - -distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }}
+          {{- end }}
           {{- range $key, $value := .Values.distributor.extraArgs }}
             - "-{{ $key }}={{ $value }}"
           {{- end }}

templates/ingester/ingester-dep.yaml

@@ -1,4 +1,4 @@
-{{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled -}}
+{{- if and (not .Values.ingester.statefulSet.enabled) .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -54,6 +54,9 @@ spec:
           args:
             - "-target=ingester"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if .Values.ingester.zoneAwareness.enabled }}
+            - "-distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }}"
+            {{- end }}
             {{- include "cortex.memcached" . | nindent 12}}
             {{- range $key, $value := .Values.ingester.extraArgs }}
             - "-{{ $key }}={{ $value }}"

templates/ingester/ingester-hpa.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled -}}
+{{- if and .Values.ingester.enabled .Values.ingester.autoscaling.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}}
 {{- with .Values.ingester.autoscaling -}}
 apiVersion: {{ include "cortex.hpaVersion" $ }}
 kind: HorizontalPodAutoscaler

templates/ingester/ingester-poddisruptionbudget.yaml

@@ -1,4 +1,4 @@
-{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) }}
+{{- if and (gt (int .Values.ingester.replicas) 1) (.Values.ingester.podDisruptionBudget) (.Values.ingester.enabled) (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) }}
 apiVersion: {{ include "cortex.pdbVersion" . }}
 kind: PodDisruptionBudget
 metadata:

templates/ingester/ingester-statefulset.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled -}}
+{{- if and .Values.ingester.statefulSet.enabled .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -110,6 +110,9 @@ spec:
           args:
             - "-target=ingester"
             - "-config.file=/etc/cortex/cortex.yaml"
+            {{- if .Values.ingester.zoneAwareness.enabled }}
+            - "-distributor.zone-awareness-enabled={{ not .Values.ingester.zoneAwareness.migration.enabled }}"
+            {{- end }}
             {{- include "cortex.memcached" . | nindent 12}}
             {{- range $key, $value := .Values.ingester.extraArgs }}
             - "-{{ $key }}={{ $value }}"

templates/ingester/ingester-svc-headless.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.ingester.enabled -}}
+{{- if and .Values.ingester.enabled (or (not .Values.ingester.zoneAwareness.enabled) .Values.ingester.zoneAwareness.migration.enabled) -}}
 apiVersion: v1
 kind: Service
 metadata: