K8SPG-647: Fix major-upgrade test

e2e-tests/run-pr.csv

@@ -1,20 +1 @@
-custom-extensions
-custom-tls
-demand-backup
-finalizers
-init-deploy
-monitoring
-monitoring-pmm3
-one-pod
-operator-self-healing
-pitr
-scaling
-scheduled-backup
-self-healing
-sidecars
-start-from-backup
-tablespaces
-telemetry-transfer
-upgrade-consistency
-upgrade-minor
-users
+major-upgrade

e2e-tests/tests/major-upgrade/01-assert.yaml

@@ -1,6 +1,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 120
+timeout: 300
 ---
 kind: StatefulSet
 apiVersion: apps/v1

e2e-tests/tests/major-upgrade/01-create-cluster.yaml

@@ -10,11 +10,10 @@ commands:
 
       get_cr \
         | yq eval '
-          .spec.postgresVersion = 12 |
-          .spec.image = "perconalab/percona-postgresql-operator:main-ppg12-postgres" |
+          .spec.postgresVersion = 13 |
+          .spec.image = "perconalab/percona-postgresql-operator:main-ppg13-postgres" |
           .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" |
-          .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-ppg12-pgbouncer" |
-          .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-ppg12-pgbackrest" |
-          .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" |
-          .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.1"}]' \
+          .spec.patroni.createReplicaMethods = ["basebackup","pgbackrest"] |
+          .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-ppg13-pgbouncer" |
+          .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-ppg13-pgbackrest"' \
         | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/major-upgrade/02-write-data.yaml

@@ -10,11 +10,7 @@ commands:
       run_psql_local \
         'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \
         "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)"
-      
+
       run_psql_local \
         '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \
         "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)"
-
-      run_psql_local \
-        '\c postgres \\\ CREATE EXTENSION pg_cron' \
-        "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)"

e2e-tests/tests/major-upgrade/20-assert.yaml

@@ -4,7 +4,20 @@ timeout: 360
 commands:
 - script: |-
     kubectl -n ${NAMESPACE} get pg,pod,job
+    #
+    # for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/pgupgrade=13-to-14 | awk '{print $1}'); do
+    #   echo "${pod} logs:"
+    #   kubectl -n ${NAMESPACE} logs ${pod}
+    # done
+    #
     sleep 5
+collectors:
+- type: pod
+  selector: "postgres-operator.crunchydata.com/data=postgres"
+  tail: 30
+- type: pod
+  selector: "postgres-operator.crunchydata.com/pgupgrade=13-to-14"
+  tail: 200
 ---
 apiVersion: pgv2.percona.com/v2
 kind: PerconaPGCluster
@@ -32,7 +45,7 @@ metadata:
     postgres-operator.crunchydata.com/cluster: major-upgrade
     postgres-operator.crunchydata.com/pgbackrest: ''
     postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
-    postgres-operator.crunchydata.com/pgbackrest-repo: repo2
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
   ownerReferences:
     - apiVersion: pgv2.percona.com/v2
       kind: PerconaPGBackup

e2e-tests/tests/major-upgrade/21-wait-and-write-data.yaml

@@ -0,0 +1,24 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - timeout: 600
+    script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      sleep 90 # wait some time for stanza to upgrade
+
+      for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+              echo "Checking the status of ${pod}:"
+              kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT version()"
+              kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT timeline_id FROM pg_control_checkpoint()"
+              kubectl -n ${NAMESPACE} exec -it ${pod} -- pgbackrest --stanza=db --log-level-console=detail check
+              kubectl -n ${NAMESPACE} exec -it ${pod} -- ls -l /pgdata/pg14_wal
+      done
+
+      run_psql_local \
+        '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' \
+        "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)"
+

e2e-tests/tests/major-upgrade/23-recreate-stanza.yaml

@@ -0,0 +1,36 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - timeout: 300
+    script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+        kubectl -n ${NAMESPACE} exec ${pod} -- touch /pgdata/sleep-forever
+        kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stop
+      done
+
+      pod=$(kubectl get -n ${NAMESPACE} pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | tail -n 1 | awk '{print $1}')
+
+      kubectl -n ${NAMESPACE} exec ${pod} -- patronictl pause
+
+      for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+        kubectl -n ${NAMESPACE} exec ${pod} -- pg_ctl -D /pgdata/pg14 stop
+      done
+
+      kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-delete
+
+      for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+        kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug start
+        kubectl -n ${NAMESPACE} exec ${pod} -- rm /pgdata/sleep-forever
+      done
+
+      kubectl -n ${NAMESPACE} exec ${pod} -- patronictl resume
+
+      # give some time to patroni for starting postgres
+      sleep 90
+
+      kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-create

e2e-tests/tests/major-upgrade/42-assert.yaml → e2e-tests/tests/major-upgrade/27-assert.yaml

@@ -6,7 +6,7 @@ kind: Job
 apiVersion: batch/v1
 metadata:
   annotations:
-    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14
   labels:
     postgres-operator.crunchydata.com/pgbackrest-backup: manual
     postgres-operator.crunchydata.com/pgbackrest-repo: repo1
@@ -21,7 +21,7 @@ status:
 apiVersion: pgv2.percona.com/v2
 kind: PerconaPGBackup
 metadata:
-  name: backup-after-15-to-16
+  name: backup-after-13-to-14
 spec:
   pgCluster: major-upgrade
   repoName: repo1

e2e-tests/tests/major-upgrade/22-run-backup.yaml → e2e-tests/tests/major-upgrade/27-run-backup.yaml

@@ -4,6 +4,6 @@ metadata:
   name: backup-after-13-to-14
 spec:
   pgCluster: major-upgrade
-  repoName: repo3
+  repoName: repo1
   options:
     - --type=full