From 4b2f983765d309133e9854aa9d4fb76151d054e3 Mon Sep 17 00:00:00 2001 From: Fabian Witt Date: Fri, 24 Jul 2026 14:37:26 +0200 Subject: [PATCH 1/3] feat(cluster): expose shutdown and failover timing knobs Expose cluster.smartShutdownTimeout, cluster.switchoverDelay, cluster.startDelay and cluster.failoverDelay, mapping to the matching CNPG Cluster spec fields. This completes the shutdown/failover timing family started by cluster.stopDelay (#941): the template never rendered these four, so setting them in values.yaml was silently dropped and they stayed pinned to the operator defaults. Motivating case: on a brief API-server connectivity loss on the node running the primary, CNPG fails the primary over. With the default smartShutdownTimeout of 180s the old primary waits for idle client connections to drain before the standby can be promoted, turning a ~1 minute blip into a multi-minute read-write outage. Lowering smartShutdownTimeout (and raising failoverDelay to ride out very short partitions) is the intended knob, but until now it required patching the CR out of band. Also covers the remaining fields requested in #523. Same wire-up as #941: values.yaml plus template block, values.schema.json regenerated with helm-schema, README regenerated with helm-docs. Verified with helm lint and helm template; failoverDelay defaults to 0 and is only rendered when set. Closes #892 Signed-off-by: Fabian Witt Assisted-by: Claude Opus 4.8 --- charts/cluster/README.md | 4 ++++ charts/cluster/templates/cluster.yaml | 12 ++++++++++++ charts/cluster/values.schema.json | 20 ++++++++++++++++++++ charts/cluster/values.yaml | 18 ++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/charts/cluster/README.md b/charts/cluster/README.md index b3d37062ad..5c45ab869e 100644 --- a/charts/cluster/README.md +++ b/charts/cluster/README.md @@ -202,6 +202,7 @@ Kubernetes: `>=1.29.0-0` | cluster.enableSuperuserAccess | bool | `true` | When this option is enabled, the operator will use the SuperuserSecret to update the postgres user password. If the secret is not present, the operator will automatically create one. When this option is disabled, the operator will ignore the SuperuserSecret content, delete it when automatically created, and then blank the password of the postgres user by setting it to NULL. | | cluster.env | list | `[]` | Env follows the Env format to pass environment variables to the pods created in the cluster | | cluster.envFrom | list | `[]` | EnvFrom follows the EnvFrom format to pass environment variables sources to the pods to be used by Env | +| cluster.failoverDelay | int | `0` | The amount of time in seconds to wait before triggering a failover after the primary instance was detected to be unhealthy. Defaults to 0 (immediate failover) when unset, per the operator default. | | cluster.imageCatalogRef | object | `{}` | Reference to `ImageCatalog` of `ClusterImageCatalog`, if specified takes precedence over `cluster.imageName` | | cluster.imageName | string | `""` | Name of the container image, supporting both tags (:) and digests for deterministic and repeatable deployments: :@sha256: | | cluster.imagePullPolicy | string | `"IfNotPresent"` | Image pull policy. One of Always, Never or IfNotPresent. If not defined, it defaults to IfNotPresent. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images | @@ -243,10 +244,13 @@ Kubernetes: `>=1.29.0-0` | cluster.securityContext | object | `{}` | Configure Container Security Context. See: https://cloudnative-pg.io/documentation/preview/security/ | | cluster.serviceAccountTemplate | object | `{}` | Configure the metadata of the generated service account | | cluster.services | object | `{}` | Customization of service definitions. Please refer to https://cloudnative-pg.io/documentation/current/service_management/ | +| cluster.smartShutdownTimeout | int | `180` | The time in seconds reserved for the smart shutdown of Postgres to complete before the operator requests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout). Defaults to 180 when unset, per the operator default. | +| cluster.startDelay | int | `3600` | The time in seconds that is allowed for a PostgreSQL instance to successfully start up. The startup probe failure threshold is derived from this value using the formula ceiling(startDelay / 10). Defaults to 3600 when unset, per the operator default. | | cluster.stopDelay | int | `1800` | The time in seconds that is allowed for the instance to wait for the shutdown to complete before being forcefully terminated. Also sets the pod's terminationGracePeriodSeconds. Defaults to 1800 (30m) when unset, per the operator default. | | cluster.storage.size | string | `"8Gi"` | | | cluster.storage.storageClass | string | `""` | | | cluster.superuserSecret | string | `""` | | +| cluster.switchoverDelay | int | `3600` | The time in seconds that is allowed for a primary instance to gracefully shut down during a switchover. Defaults to 3600 (1h) when unset, per the operator default. | | cluster.walStorage.enabled | bool | `false` | | | cluster.walStorage.size | string | `"1Gi"` | | | cluster.walStorage.storageClass | string | `""` | | diff --git a/charts/cluster/templates/cluster.yaml b/charts/cluster/templates/cluster.yaml index ecf04b6de2..9da371a55c 100644 --- a/charts/cluster/templates/cluster.yaml +++ b/charts/cluster/templates/cluster.yaml @@ -56,6 +56,18 @@ spec: {{- with .Values.cluster.stopDelay }} stopDelay: {{ . }} {{- end }} + {{- with .Values.cluster.smartShutdownTimeout }} + smartShutdownTimeout: {{ . }} + {{- end }} + {{- with .Values.cluster.switchoverDelay }} + switchoverDelay: {{ . }} + {{- end }} + {{- with .Values.cluster.startDelay }} + startDelay: {{ . }} + {{- end }} + {{- with .Values.cluster.failoverDelay }} + failoverDelay: {{ . }} + {{- end }} logLevel: {{ .Values.cluster.logLevel }} {{- with .Values.cluster.certificates }} certificates: diff --git a/charts/cluster/values.schema.json b/charts/cluster/values.schema.json index cf581a8577..d41befb905 100644 --- a/charts/cluster/values.schema.json +++ b/charts/cluster/values.schema.json @@ -318,6 +318,11 @@ }, "type": "array" }, + "failoverDelay": { + "default": 0, + "description": "The amount of time in seconds to wait before triggering a failover after the primary instance was\ndetected to be unhealthy. Defaults to 0 (immediate failover) when unset, per the operator default.", + "type": "integer" + }, "imageCatalogRef": { "description": "Reference to `ImageCatalog` of `ClusterImageCatalog`, if specified takes precedence over `cluster.imageName`", "required": [], @@ -582,6 +587,16 @@ "required": [], "type": "object" }, + "smartShutdownTimeout": { + "default": 180, + "description": "The time in seconds reserved for the smart shutdown of Postgres to complete before the operator\nrequests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout).\nDefaults to 180 when unset, per the operator default.", + "type": "integer" + }, + "startDelay": { + "default": 3600, + "description": "The time in seconds that is allowed for a PostgreSQL instance to successfully start up. The startup\nprobe failure threshold is derived from this value using the formula ceiling(startDelay / 10).\nDefaults to 3600 when unset, per the operator default.", + "type": "integer" + }, "stopDelay": { "default": 1800, "description": "The time in seconds that is allowed for the instance to wait for the shutdown\nto complete before being forcefully terminated. Also sets the pod's terminationGracePeriodSeconds.\nDefaults to 1800 (30m) when unset, per the operator default.", @@ -605,6 +620,11 @@ "default": "", "type": "string" }, + "switchoverDelay": { + "default": 3600, + "description": "The time in seconds that is allowed for a primary instance to gracefully shut down during a switchover.\nDefaults to 3600 (1h) when unset, per the operator default.", + "type": "integer" + }, "walStorage": { "properties": { "enabled": { diff --git a/charts/cluster/values.yaml b/charts/cluster/values.yaml index a85aa39f9b..57d00a0b47 100644 --- a/charts/cluster/values.yaml +++ b/charts/cluster/values.yaml @@ -258,6 +258,24 @@ cluster: # Defaults to 1800 (30m) when unset, per the operator default. stopDelay: 1800 + # -- The time in seconds reserved for the smart shutdown of Postgres to complete before the operator + # requests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout). + # Defaults to 180 when unset, per the operator default. + smartShutdownTimeout: 180 + + # -- The time in seconds that is allowed for a primary instance to gracefully shut down during a switchover. + # Defaults to 3600 (1h) when unset, per the operator default. + switchoverDelay: 3600 + + # -- The time in seconds that is allowed for a PostgreSQL instance to successfully start up. The startup + # probe failure threshold is derived from this value using the formula ceiling(startDelay / 10). + # Defaults to 3600 when unset, per the operator default. + startDelay: 3600 + + # -- The amount of time in seconds to wait before triggering a failover after the primary instance was + # detected to be unhealthy. Defaults to 0 (immediate failover) when unset, per the operator default. + failoverDelay: 0 + # -- The instances' log level, one of the following values: error, warning, info (default), debug, trace logLevel: "info" From 697dbf0e40ae7dd8ba3d3fb7d41a2c18f5d9884f Mon Sep 17 00:00:00 2001 From: Fabian Witt Date: Mon, 14 Sep 2026 09:42:46 +0200 Subject: [PATCH 2/3] fix(cluster): render smartShutdownTimeout when set to 0 smartShutdownTimeout is the one field in this timing family where 0 is a distinct, meaningful value. It is a *int32 in the CNPG API, and GetSmartShutdownTimeout() returns the pointed-to value whenever it is set, so 0 makes the operator skip the smart shutdown phase and request a fast shutdown right away. The {{- with }} guard treats 0 as empty and drops the key, so smartShutdownTimeout: 0 silently fell back to the operator default of 180. Guard on key presence instead. The other fields keep {{- with }}: startDelay, stopDelay and switchoverDelay have `> 0` fallbacks in their own getters, and failoverDelay defaults to 0, so there a dropped 0 and the operator default are the same thing. values.yaml now documents the 0 case; README.md and values.schema.json regenerated with the pinned helm-docs v1.14.2 and helm-schema 0.23.4. Verified with helm lint and helm template: 0, 30 and the default 180 all render, an explicit null leaves the key unset. Spotted by @juzu5 in review of #972. Signed-off-by: Fabian Witt Assisted-by: Claude Opus 5 --- charts/cluster/README.md | 2 +- charts/cluster/templates/cluster.yaml | 5 +++-- charts/cluster/values.schema.json | 2 +- charts/cluster/values.yaml | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/charts/cluster/README.md b/charts/cluster/README.md index 5c45ab869e..6dd8ce55df 100644 --- a/charts/cluster/README.md +++ b/charts/cluster/README.md @@ -244,7 +244,7 @@ Kubernetes: `>=1.29.0-0` | cluster.securityContext | object | `{}` | Configure Container Security Context. See: https://cloudnative-pg.io/documentation/preview/security/ | | cluster.serviceAccountTemplate | object | `{}` | Configure the metadata of the generated service account | | cluster.services | object | `{}` | Customization of service definitions. Please refer to https://cloudnative-pg.io/documentation/current/service_management/ | -| cluster.smartShutdownTimeout | int | `180` | The time in seconds reserved for the smart shutdown of Postgres to complete before the operator requests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout). Defaults to 180 when unset, per the operator default. | +| cluster.smartShutdownTimeout | int | `180` | The time in seconds reserved for the smart shutdown of Postgres to complete before the operator requests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout). Set to 0 to skip the smart phase entirely and request a fast shutdown right away. Defaults to 180 when unset, per the operator default. | | cluster.startDelay | int | `3600` | The time in seconds that is allowed for a PostgreSQL instance to successfully start up. The startup probe failure threshold is derived from this value using the formula ceiling(startDelay / 10). Defaults to 3600 when unset, per the operator default. | | cluster.stopDelay | int | `1800` | The time in seconds that is allowed for the instance to wait for the shutdown to complete before being forcefully terminated. Also sets the pod's terminationGracePeriodSeconds. Defaults to 1800 (30m) when unset, per the operator default. | | cluster.storage.size | string | `"8Gi"` | | diff --git a/charts/cluster/templates/cluster.yaml b/charts/cluster/templates/cluster.yaml index 9da371a55c..872dacca73 100644 --- a/charts/cluster/templates/cluster.yaml +++ b/charts/cluster/templates/cluster.yaml @@ -56,8 +56,9 @@ spec: {{- with .Values.cluster.stopDelay }} stopDelay: {{ . }} {{- end }} - {{- with .Values.cluster.smartShutdownTimeout }} - smartShutdownTimeout: {{ . }} + {{- /* smartShutdownTimeout accepts 0 (skip the smart phase), which `with` would drop, so test for the key */}} + {{- if hasKey .Values.cluster "smartShutdownTimeout" }} + smartShutdownTimeout: {{ .Values.cluster.smartShutdownTimeout }} {{- end }} {{- with .Values.cluster.switchoverDelay }} switchoverDelay: {{ . }} diff --git a/charts/cluster/values.schema.json b/charts/cluster/values.schema.json index d41befb905..caf90a7cd0 100644 --- a/charts/cluster/values.schema.json +++ b/charts/cluster/values.schema.json @@ -589,7 +589,7 @@ }, "smartShutdownTimeout": { "default": 180, - "description": "The time in seconds reserved for the smart shutdown of Postgres to complete before the operator\nrequests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout).\nDefaults to 180 when unset, per the operator default.", + "description": "The time in seconds reserved for the smart shutdown of Postgres to complete before the operator\nrequests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout).\nSet to 0 to skip the smart phase entirely and request a fast shutdown right away.\nDefaults to 180 when unset, per the operator default.", "type": "integer" }, "startDelay": { diff --git a/charts/cluster/values.yaml b/charts/cluster/values.yaml index 57d00a0b47..b6d923f329 100644 --- a/charts/cluster/values.yaml +++ b/charts/cluster/values.yaml @@ -260,6 +260,7 @@ cluster: # -- The time in seconds reserved for the smart shutdown of Postgres to complete before the operator # requests a fast shutdown. Reserve enough time for the fast phase (that is: stopDelay - smartShutdownTimeout). + # Set to 0 to skip the smart phase entirely and request a fast shutdown right away. # Defaults to 180 when unset, per the operator default. smartShutdownTimeout: 180 From 570c48f16a84c4063445d538a936e3786221e838 Mon Sep 17 00:00:00 2001 From: Fabian Witt Date: Mon, 14 Sep 2026 18:51:19 +0200 Subject: [PATCH 3/3] test(cluster): cover the shutdown and failover timing knobs Extend postgresql-cluster-configuration with stopDelay, smartShutdownTimeout, switchoverDelay, startDelay and failoverDelay, as TESTING.md asks for whenever a feature is added, and add a dedicated case for smartShutdownTimeout: 0. The zero case needs its own cluster because it is exactly the value the template used to lose: `with` treats 0 as empty, drops the key, and the CRD default puts 180 back. Asserting smartShutdownTimeout: 0 therefore fails against the pre-fix template, which makes it a regression guard rather than a restatement of the happy path. Checked both ways with helm template. stopDelay (#941) had no coverage either, so it comes along with the rest of the family. Signed-off-by: Fabian Witt Assisted-by: Claude Opus 5 --- ..._default_configuration_cluster-assert.yaml | 5 +++++ .../01-non_default_configuration_cluster.yaml | 5 +++++ ...04-zero_smart_shutdown_timeout-assert.yaml | 6 ++++++ .../04-zero_smart_shutdown_timeout.yaml | 19 +++++++++++++++++ .../chainsaw-test.yaml | 21 +++++++++++++++++++ 5 files changed, 56 insertions(+) create mode 100644 charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout-assert.yaml create mode 100644 charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout.yaml diff --git a/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster-assert.yaml b/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster-assert.yaml index 9d899a0237..3a35f1091a 100644 --- a/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster-assert.yaml +++ b/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster-assert.yaml @@ -113,6 +113,11 @@ spec: priorityClassName: mega-high primaryUpdateStrategy: supervised primaryUpdateMethod: restart + stopDelay: 900 + smartShutdownTimeout: 120 + switchoverDelay: 60 + startDelay: 120 + failoverDelay: 30 logLevel: warning managed: roles: diff --git a/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster.yaml b/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster.yaml index 13805d3335..0e153e7417 100644 --- a/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster.yaml +++ b/charts/cluster/test/postgresql-cluster-configuration/01-non_default_configuration_cluster.yaml @@ -25,6 +25,11 @@ cluster: priorityClassName: mega-high primaryUpdateMethod: restart primaryUpdateStrategy: supervised + stopDelay: 900 + smartShutdownTimeout: 120 + switchoverDelay: 60 + startDelay: 120 + failoverDelay: 30 logLevel: warning affinity: topologyKey: kubernetes.io/hostname diff --git a/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout-assert.yaml b/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout-assert.yaml new file mode 100644 index 0000000000..4872efc91d --- /dev/null +++ b/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: zero-smart-shutdown-timeout-cluster +spec: + smartShutdownTimeout: 0 diff --git a/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout.yaml b/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout.yaml new file mode 100644 index 0000000000..54ec5a31a4 --- /dev/null +++ b/charts/cluster/test/postgresql-cluster-configuration/04-zero_smart_shutdown_timeout.yaml @@ -0,0 +1,19 @@ +## +# `smartShutdownTimeout: 0` is a valid setting that makes the operator skip the +# smart shutdown phase entirely. It differs from the operator default of 180, so +# it has to survive templating: a `with` guard treats 0 as empty, drops the key, +# and the CRD default silently puts it back at 180. +# Same non-runnable image as 01 — this only verifies spec propagation. +type: postgresql +mode: standalone + +cluster: + instances: 1 + imageName: ghcr.io/cloudnative-pg/crazycustomimage:99.99 + storage: + size: 256Mi + storageClass: standard + smartShutdownTimeout: 0 + +backups: + enabled: false diff --git a/charts/cluster/test/postgresql-cluster-configuration/chainsaw-test.yaml b/charts/cluster/test/postgresql-cluster-configuration/chainsaw-test.yaml index 9eeb457a93..1ed4841f8d 100644 --- a/charts/cluster/test/postgresql-cluster-configuration/chainsaw-test.yaml +++ b/charts/cluster/test/postgresql-cluster-configuration/chainsaw-test.yaml @@ -47,6 +47,26 @@ spec: recovery-backup-database-owner ../../ - assert: file: ./03-recovery_backup_database_owner-assert.yaml + - name: Install a cluster with smartShutdownTimeout set to zero + description: | + 0 is a meaningful value for smartShutdownTimeout (skip the smart shutdown + phase) and differs from the operator default of 180, so the template must + not drop it. + try: + - script: + content: | + helm upgrade \ + --install \ + --namespace $NAMESPACE \ + --values ./04-zero_smart_shutdown_timeout.yaml \ + --wait \ + zero-smart-shutdown-timeout ../../ + - assert: + file: ./04-zero_smart_shutdown_timeout-assert.yaml + catch: + - describe: + apiVersion: postgresql.cnpg.io/v1 + kind: Cluster - name: Cleanup try: - script: @@ -54,3 +74,4 @@ spec: helm uninstall --namespace $NAMESPACE non-default-configuration helm uninstall --namespace $NAMESPACE recovery-object-store-database-owner helm uninstall --namespace $NAMESPACE recovery-backup-database-owner + helm uninstall --namespace $NAMESPACE zero-smart-shutdown-timeout