diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index 8c0ba3867f..6b2bf90256 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -1,20 +1 @@ -custom-extensions -custom-tls -demand-backup -finalizers -init-deploy -monitoring -monitoring-pmm3 -one-pod -operator-self-healing -pitr -scaling -scheduled-backup -self-healing -sidecars -start-from-backup -tablespaces -telemetry-transfer -upgrade-consistency -upgrade-minor -users +major-upgrade diff --git a/e2e-tests/tests/major-upgrade/01-assert.yaml b/e2e-tests/tests/major-upgrade/01-assert.yaml index 8fee6151e6..b580d781dc 100644 --- a/e2e-tests/tests/major-upgrade/01-assert.yaml +++ b/e2e-tests/tests/major-upgrade/01-assert.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 120 +timeout: 300 --- kind: StatefulSet apiVersion: apps/v1 diff --git a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml index 729fbcc51f..ef9f6ffa0f 100644 --- a/e2e-tests/tests/major-upgrade/01-create-cluster.yaml +++ b/e2e-tests/tests/major-upgrade/01-create-cluster.yaml @@ -10,11 +10,10 @@ commands: get_cr \ | yq eval ' - .spec.postgresVersion = 12 | - .spec.image = "perconalab/percona-postgresql-operator:main-ppg12-postgres" | + .spec.postgresVersion = 13 | + .spec.image = "perconalab/percona-postgresql-operator:main-ppg13-postgres" | .spec.instances[0].dataVolumeClaimSpec.resources.requests.storage = "3Gi" | - .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-ppg12-pgbouncer" | - .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-ppg12-pgbackrest" | - .spec.patroni.dynamicConfiguration.postgresql.parameters.shared_preload_libraries = "pg_cron" | - .spec.extensions.custom += [{"name": "pg_cron", "version": "1.6.1"}]' \ + .spec.patroni.createReplicaMethods = ["basebackup","pgbackrest"] | + .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-ppg13-pgbouncer" | + .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-ppg13-pgbackrest"' \ | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/major-upgrade/02-write-data.yaml b/e2e-tests/tests/major-upgrade/02-write-data.yaml index da5b97f2a7..9dab40255e 100644 --- a/e2e-tests/tests/major-upgrade/02-write-data.yaml +++ b/e2e-tests/tests/major-upgrade/02-write-data.yaml @@ -10,11 +10,7 @@ commands: run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" - + run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" - - run_psql_local \ - '\c postgres \\\ CREATE EXTENSION pg_cron' \ - "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" \ No newline at end of file diff --git a/e2e-tests/tests/major-upgrade/10-12-to-13.yaml b/e2e-tests/tests/major-upgrade/10-12-to-13.yaml deleted file mode 100644 index 96a7fe9207..0000000000 --- a/e2e-tests/tests/major-upgrade/10-12-to-13.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - source ../../functions - - kubectl apply -n ${NAMESPACE} -f - <<-EOF - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGUpgrade - metadata: - name: 12-to-13 - spec: - postgresClusterName: major-upgrade - image: ${IMAGE_UPGRADE} - fromPostgresVersion: 12 - toPostgresVersion: 13 - toPostgresImage: $(get_container_image "postgres" 13) - toPgBouncerImage: $(get_container_image "pgbouncer" 13) - toPgBackRestImage: $(get_container_image "pgbackrest" 13) - EOF - diff --git a/e2e-tests/tests/major-upgrade/11-assert.yaml b/e2e-tests/tests/major-upgrade/11-assert.yaml deleted file mode 100644 index 47ccba03b5..0000000000 --- a/e2e-tests/tests/major-upgrade/11-assert.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 -commands: -- script: |- - kubectl get postgrescluster major-upgrade \ - -n ${NAMESPACE} \ - -o yaml \ - | yq eval '.status.pgbackrest.repos' - ---- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: major-upgrade -status: - pgbackrest: - repos: - - bound: true - name: repo2 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/11-change-repo.yaml b/e2e-tests/tests/major-upgrade/11-change-repo.yaml deleted file mode 100644 index 22094b19f4..0000000000 --- a/e2e-tests/tests/major-upgrade/11-change-repo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - kubectl patch \ - -n $NAMESPACE \ - perconapgcluster major-upgrade \ - --type='json' \ - -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]' diff --git a/e2e-tests/tests/major-upgrade/12-assert.yaml b/e2e-tests/tests/major-upgrade/12-assert.yaml deleted file mode 100644 index d1790eabf5..0000000000 --- a/e2e-tests/tests/major-upgrade/12-assert.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 ---- -kind: Job -apiVersion: batch/v1 -metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13 - labels: - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo2 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true -status: - succeeded: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo2 - options: - - --type=full -status: - state: Succeeded diff --git a/e2e-tests/tests/major-upgrade/12-run-backup.yaml b/e2e-tests/tests/major-upgrade/12-run-backup.yaml deleted file mode 100644 index 601008d4ca..0000000000 --- a/e2e-tests/tests/major-upgrade/12-run-backup.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGBackup -metadata: - name: backup-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo2 - options: - - --type=full diff --git a/e2e-tests/tests/major-upgrade/13-assert.yaml b/e2e-tests/tests/major-upgrade/13-assert.yaml deleted file mode 100644 index 2d05bb7d88..0000000000 --- a/e2e-tests/tests/major-upgrade/13-assert.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 720 -commands: -- script: |- - set -o errexit - - kubectl -n ${NAMESPACE} get pod - - for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do - phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) - if [[ "${phase}" != "Running" ]]; then - echo "Waiting for ${pod} to start running" - continue - fi - echo "PostgreSQL logs from ${pod}:" - echo "find /pgdata/pg13/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ - | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null - done - - sleep 30 -collectors: -- type: pod - selector: "postgres-operator.crunchydata.com/data=postgres" - tail: 30 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore -metadata: - name: restore-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo2 -status: - state: Succeeded ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: major-upgrade -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready diff --git a/e2e-tests/tests/major-upgrade/13-run-restore.yaml b/e2e-tests/tests/major-upgrade/13-run-restore.yaml deleted file mode 100644 index bba96fe985..0000000000 --- a/e2e-tests/tests/major-upgrade/13-run-restore.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGRestore -metadata: - name: restore-after-12-to-13 -spec: - pgCluster: major-upgrade - repoName: repo2 diff --git a/e2e-tests/tests/major-upgrade/20-assert.yaml b/e2e-tests/tests/major-upgrade/20-assert.yaml index c506a745f5..2cbe8bad8d 100644 --- a/e2e-tests/tests/major-upgrade/20-assert.yaml +++ b/e2e-tests/tests/major-upgrade/20-assert.yaml @@ -4,7 +4,20 @@ timeout: 360 commands: - script: |- kubectl -n ${NAMESPACE} get pg,pod,job + # + # for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/pgupgrade=13-to-14 | awk '{print $1}'); do + # echo "${pod} logs:" + # kubectl -n ${NAMESPACE} logs ${pod} + # done + # sleep 5 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 +- type: pod + selector: "postgres-operator.crunchydata.com/pgupgrade=13-to-14" + tail: 200 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -32,7 +45,7 @@ metadata: postgres-operator.crunchydata.com/cluster: major-upgrade postgres-operator.crunchydata.com/pgbackrest: '' postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo2 + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup diff --git a/e2e-tests/tests/major-upgrade/21-assert.yaml b/e2e-tests/tests/major-upgrade/21-assert.yaml deleted file mode 100644 index 301240b0f5..0000000000 --- a/e2e-tests/tests/major-upgrade/21-assert.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 -commands: -- script: |- - kubectl get postgrescluster major-upgrade \ - -n ${NAMESPACE} \ - -o yaml \ - | yq eval '.status.pgbackrest.repos' - ---- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: major-upgrade -status: - pgbackrest: - repos: - - bound: true - name: repo3 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/21-change-repo.yaml b/e2e-tests/tests/major-upgrade/21-change-repo.yaml deleted file mode 100644 index 57f9edc8a4..0000000000 --- a/e2e-tests/tests/major-upgrade/21-change-repo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - kubectl patch \ - -n $NAMESPACE \ - perconapgcluster major-upgrade \ - --type='json' \ - -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo3"}]' diff --git a/e2e-tests/tests/major-upgrade/21-wait-and-write-data.yaml b/e2e-tests/tests/major-upgrade/21-wait-and-write-data.yaml new file mode 100644 index 0000000000..5310fcb884 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/21-wait-and-write-data.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 90 # wait some time for stanza to upgrade + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + echo "Checking the status of ${pod}:" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT version()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT timeline_id FROM pg_control_checkpoint()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- pgbackrest --stanza=db --log-level-console=detail check + kubectl -n ${NAMESPACE} exec -it ${pod} -- ls -l /pgdata/pg14_wal + done + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501)' \ + "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" + diff --git a/e2e-tests/tests/major-upgrade/23-recreate-stanza.yaml b/e2e-tests/tests/major-upgrade/23-recreate-stanza.yaml new file mode 100644 index 0000000000..1e38f9309e --- /dev/null +++ b/e2e-tests/tests/major-upgrade/23-recreate-stanza.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 300 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- touch /pgdata/sleep-forever + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stop + done + + pod=$(kubectl get -n ${NAMESPACE} pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | tail -n 1 | awk '{print $1}') + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl pause + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pg_ctl -D /pgdata/pg14 stop + done + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-delete + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug start + kubectl -n ${NAMESPACE} exec ${pod} -- rm /pgdata/sleep-forever + done + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl resume + + # give some time to patroni for starting postgres + sleep 90 + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-create diff --git a/e2e-tests/tests/major-upgrade/42-assert.yaml b/e2e-tests/tests/major-upgrade/27-assert.yaml similarity index 93% rename from e2e-tests/tests/major-upgrade/42-assert.yaml rename to e2e-tests/tests/major-upgrade/27-assert.yaml index 2b003b69a9..788d3de044 100644 --- a/e2e-tests/tests/major-upgrade/42-assert.yaml +++ b/e2e-tests/tests/major-upgrade/27-assert.yaml @@ -6,7 +6,7 @@ kind: Job apiVersion: batch/v1 metadata: annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 labels: postgres-operator.crunchydata.com/pgbackrest-backup: manual postgres-operator.crunchydata.com/pgbackrest-repo: repo1 @@ -21,7 +21,7 @@ status: apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup metadata: - name: backup-after-15-to-16 + name: backup-after-13-to-14 spec: pgCluster: major-upgrade repoName: repo1 diff --git a/e2e-tests/tests/major-upgrade/22-run-backup.yaml b/e2e-tests/tests/major-upgrade/27-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/22-run-backup.yaml rename to e2e-tests/tests/major-upgrade/27-run-backup.yaml index a7a1050c53..7f60f47386 100644 --- a/e2e-tests/tests/major-upgrade/22-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/27-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo3 + repoName: repo1 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/23-assert.yaml b/e2e-tests/tests/major-upgrade/28-assert.yaml similarity index 79% rename from e2e-tests/tests/major-upgrade/23-assert.yaml rename to e2e-tests/tests/major-upgrade/28-assert.yaml index 75b542e689..ad1b782f7d 100644 --- a/e2e-tests/tests/major-upgrade/23-assert.yaml +++ b/e2e-tests/tests/major-upgrade/28-assert.yaml @@ -7,6 +7,11 @@ commands: kubectl -n ${NAMESPACE} get pod + # for pod in $(kubectl -n ${NAMESPACE} get pod -l job-name=major-upgrade-pgbackrest-restore | awk '{print $1}'); do + # echo "${pod} logs:" + # kubectl -n ${NAMESPACE} logs ${pod} + # done + # for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) if [[ "${phase}" != "Running" ]]; then @@ -18,11 +23,14 @@ commands: | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done - sleep 30 + sleep 15 collectors: - type: pod selector: "postgres-operator.crunchydata.com/data=postgres" tail: 30 +- type: pod + selector: "job-name=major-upgrade-pgbackrest-restore" + tail: 300 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGRestore @@ -30,7 +38,7 @@ metadata: name: restore-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo3 + repoName: repo1 status: state: Succeeded --- diff --git a/e2e-tests/tests/major-upgrade/23-run-restore.yaml b/e2e-tests/tests/major-upgrade/28-run-restore.yaml similarity index 68% rename from e2e-tests/tests/major-upgrade/23-run-restore.yaml rename to e2e-tests/tests/major-upgrade/28-run-restore.yaml index 079ebf7b77..5da4967f36 100644 --- a/e2e-tests/tests/major-upgrade/23-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/28-run-restore.yaml @@ -4,4 +4,6 @@ metadata: name: restore-after-13-to-14 spec: pgCluster: major-upgrade - repoName: repo3 + repoName: repo1 + options: + - --log-level-console=debug diff --git a/e2e-tests/tests/major-upgrade/34-assert.yaml b/e2e-tests/tests/major-upgrade/29-assert.yaml similarity index 62% rename from e2e-tests/tests/major-upgrade/34-assert.yaml rename to e2e-tests/tests/major-upgrade/29-assert.yaml index 7321c61abe..bf53e293ae 100644 --- a/e2e-tests/tests/major-upgrade/34-assert.yaml +++ b/e2e-tests/tests/major-upgrade/29-assert.yaml @@ -5,6 +5,8 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 09-read-from-primary + name: 29-read-from-primary data: - data: ' 100500' + data: |2- + 100500 + 100501 diff --git a/e2e-tests/tests/major-upgrade/14-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/29-read-from-primary.yaml similarity index 85% rename from e2e-tests/tests/major-upgrade/14-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/29-read-from-primary.yaml index ac80bb337c..ebf6753f9b 100644 --- a/e2e-tests/tests/major-upgrade/14-read-from-primary.yaml +++ b/e2e-tests/tests/major-upgrade/29-read-from-primary.yaml @@ -9,4 +9,4 @@ commands: source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 05-read-from-primary --from-literal=data="${data}" + kubectl create configmap -n "${NAMESPACE}" 29-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/major-upgrade/30-assert.yaml b/e2e-tests/tests/major-upgrade/30-assert.yaml index b871b4faf8..edf3605d1b 100644 --- a/e2e-tests/tests/major-upgrade/30-assert.yaml +++ b/e2e-tests/tests/major-upgrade/30-assert.yaml @@ -5,6 +5,13 @@ commands: - script: |- kubectl -n ${NAMESPACE} get pg,pod,job sleep 5 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 +- type: pod + selector: "postgres-operator.crunchydata.com/pgupgrade=13-to-14" + tail: 200 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -32,7 +39,7 @@ metadata: postgres-operator.crunchydata.com/cluster: major-upgrade postgres-operator.crunchydata.com/pgbackrest: '' postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup diff --git a/e2e-tests/tests/major-upgrade/31-assert.yaml b/e2e-tests/tests/major-upgrade/31-assert.yaml deleted file mode 100644 index 6af985e4b7..0000000000 --- a/e2e-tests/tests/major-upgrade/31-assert.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 -commands: -- script: |- - kubectl get postgrescluster major-upgrade \ - -n ${NAMESPACE} \ - -o yaml \ - | yq eval '.status.pgbackrest.repos' - ---- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: major-upgrade -status: - pgbackrest: - repos: - - bound: true - name: repo4 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/31-change-repo.yaml b/e2e-tests/tests/major-upgrade/31-change-repo.yaml deleted file mode 100644 index 33711377c4..0000000000 --- a/e2e-tests/tests/major-upgrade/31-change-repo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - kubectl patch \ - -n $NAMESPACE \ - perconapgcluster major-upgrade \ - --type='json' \ - -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo4"}]' diff --git a/e2e-tests/tests/major-upgrade/31-wait-and-write-data.yaml b/e2e-tests/tests/major-upgrade/31-wait-and-write-data.yaml new file mode 100644 index 0000000000..0f5381c29d --- /dev/null +++ b/e2e-tests/tests/major-upgrade/31-wait-and-write-data.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 90 + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + echo "Checking the status of ${pod}:" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT version()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT timeline_id FROM pg_control_checkpoint()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- pgbackrest --stanza=db --log-level-console=detail check + kubectl -n ${NAMESPACE} exec -it ${pod} -- ls -l /pgdata/pg15_wal + done + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100502)' \ + "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" + diff --git a/e2e-tests/tests/major-upgrade/33-recreate-stanza.yaml b/e2e-tests/tests/major-upgrade/33-recreate-stanza.yaml new file mode 100644 index 0000000000..d2aaa59ab1 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/33-recreate-stanza.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 300 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- touch /pgdata/sleep-forever + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stop + done + + pod=$(kubectl get -n ${NAMESPACE} pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | tail -n 1 | awk '{print $1}') + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl pause + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pg_ctl -D /pgdata/pg15 stop + done + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-delete + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug start + kubectl -n ${NAMESPACE} exec ${pod} -- rm /pgdata/sleep-forever + done + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl resume + + # give some time to patroni for starting postgres + sleep 90 + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-create diff --git a/e2e-tests/tests/major-upgrade/32-assert.yaml b/e2e-tests/tests/major-upgrade/37-assert.yaml similarity index 88% rename from e2e-tests/tests/major-upgrade/32-assert.yaml rename to e2e-tests/tests/major-upgrade/37-assert.yaml index 75daf8364c..14ca1327c2 100644 --- a/e2e-tests/tests/major-upgrade/32-assert.yaml +++ b/e2e-tests/tests/major-upgrade/37-assert.yaml @@ -9,7 +9,7 @@ metadata: postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-14-to-15 labels: postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup @@ -24,7 +24,7 @@ metadata: name: backup-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo4 + repoName: repo1 options: - --type=full status: diff --git a/e2e-tests/tests/major-upgrade/32-run-backup.yaml b/e2e-tests/tests/major-upgrade/37-run-backup.yaml similarity index 89% rename from e2e-tests/tests/major-upgrade/32-run-backup.yaml rename to e2e-tests/tests/major-upgrade/37-run-backup.yaml index 792209c20c..e787fedfca 100644 --- a/e2e-tests/tests/major-upgrade/32-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/37-run-backup.yaml @@ -4,6 +4,6 @@ metadata: name: backup-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo4 + repoName: repo1 options: - --type=full diff --git a/e2e-tests/tests/major-upgrade/33-assert.yaml b/e2e-tests/tests/major-upgrade/38-assert.yaml similarity index 92% rename from e2e-tests/tests/major-upgrade/33-assert.yaml rename to e2e-tests/tests/major-upgrade/38-assert.yaml index 1d3a8a9aae..965f474253 100644 --- a/e2e-tests/tests/major-upgrade/33-assert.yaml +++ b/e2e-tests/tests/major-upgrade/38-assert.yaml @@ -18,11 +18,14 @@ commands: | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done - sleep 30 + sleep 15 collectors: - type: pod selector: "postgres-operator.crunchydata.com/data=postgres" tail: 30 +- type: pod + selector: "job-name=major-upgrade-pgbackrest-restore" + tail: 300 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGRestore @@ -30,7 +33,7 @@ metadata: name: restore-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo4 + repoName: repo1 status: state: Succeeded --- diff --git a/e2e-tests/tests/major-upgrade/33-run-restore.yaml b/e2e-tests/tests/major-upgrade/38-run-restore.yaml similarity index 68% rename from e2e-tests/tests/major-upgrade/33-run-restore.yaml rename to e2e-tests/tests/major-upgrade/38-run-restore.yaml index 8e7c7c6dab..8bf3986aab 100644 --- a/e2e-tests/tests/major-upgrade/33-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/38-run-restore.yaml @@ -4,5 +4,6 @@ metadata: name: restore-after-14-to-15 spec: pgCluster: major-upgrade - repoName: repo4 - + repoName: repo1 + options: + - --log-level-console=debug diff --git a/e2e-tests/tests/major-upgrade/24-assert.yaml b/e2e-tests/tests/major-upgrade/39-assert.yaml similarity index 58% rename from e2e-tests/tests/major-upgrade/24-assert.yaml rename to e2e-tests/tests/major-upgrade/39-assert.yaml index c86dbb55fc..8bf192f7d0 100644 --- a/e2e-tests/tests/major-upgrade/24-assert.yaml +++ b/e2e-tests/tests/major-upgrade/39-assert.yaml @@ -5,6 +5,9 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 07-read-from-primary + name: 39-read-from-primary data: - data: ' 100500' + data: |2- + 100500 + 100501 + 100502 diff --git a/e2e-tests/tests/major-upgrade/24-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/39-read-from-primary.yaml similarity index 76% rename from e2e-tests/tests/major-upgrade/24-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/39-read-from-primary.yaml index bfa5e93458..80a47927b8 100644 --- a/e2e-tests/tests/major-upgrade/24-read-from-primary.yaml +++ b/e2e-tests/tests/major-upgrade/39-read-from-primary.yaml @@ -9,4 +9,4 @@ commands: source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 07-read-from-primary --from-literal=data="${data}" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 39-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/major-upgrade/40-assert.yaml b/e2e-tests/tests/major-upgrade/40-assert.yaml index de3a338c83..45a1e643b9 100644 --- a/e2e-tests/tests/major-upgrade/40-assert.yaml +++ b/e2e-tests/tests/major-upgrade/40-assert.yaml @@ -5,6 +5,13 @@ commands: - script: |- kubectl -n ${NAMESPACE} get pg,pod,job sleep 5 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 +- type: pod + selector: "postgres-operator.crunchydata.com/pgupgrade=13-to-14" + tail: 200 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster @@ -32,7 +39,7 @@ metadata: postgres-operator.crunchydata.com/cluster: major-upgrade postgres-operator.crunchydata.com/pgbackrest: '' postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo4 + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup diff --git a/e2e-tests/tests/major-upgrade/41-assert.yaml b/e2e-tests/tests/major-upgrade/41-assert.yaml deleted file mode 100644 index 1b95780c8f..0000000000 --- a/e2e-tests/tests/major-upgrade/41-assert.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 -commands: -- script: |- - kubectl get postgrescluster major-upgrade \ - -n ${NAMESPACE} \ - -o yaml \ - | yq eval '.status.pgbackrest.repos' - ---- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: major-upgrade -status: - pgbackrest: - repos: - - bound: true - name: repo1 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/41-change-repo.yaml b/e2e-tests/tests/major-upgrade/41-change-repo.yaml deleted file mode 100644 index d3d277599f..0000000000 --- a/e2e-tests/tests/major-upgrade/41-change-repo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - kubectl patch \ - -n $NAMESPACE \ - perconapgcluster major-upgrade \ - --type='json' \ - -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo1"}]' diff --git a/e2e-tests/tests/major-upgrade/41-wait-and-write-data.yaml b/e2e-tests/tests/major-upgrade/41-wait-and-write-data.yaml new file mode 100644 index 0000000000..45cbb360c8 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/41-wait-and-write-data.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 90 + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + echo "Checking the status of ${pod}:" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT version()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT timeline_id FROM pg_control_checkpoint()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- pgbackrest --stanza=db --log-level-console=detail check + kubectl -n ${NAMESPACE} exec -it ${pod} -- ls -l /pgdata/pg16_wal + done + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100503)' \ + "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" + diff --git a/e2e-tests/tests/major-upgrade/43-recreate-stanza.yaml b/e2e-tests/tests/major-upgrade/43-recreate-stanza.yaml new file mode 100644 index 0000000000..58b18bc4cd --- /dev/null +++ b/e2e-tests/tests/major-upgrade/43-recreate-stanza.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 300 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- touch /pgdata/sleep-forever + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stop + done + + pod=$(kubectl get -n ${NAMESPACE} pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | tail -n 1 | awk '{print $1}') + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl pause + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pg_ctl -D /pgdata/pg16 stop + done + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-delete + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug start + kubectl -n ${NAMESPACE} exec ${pod} -- rm /pgdata/sleep-forever + done + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl resume + + # give some time to patroni for starting postgres + sleep 90 + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-create diff --git a/e2e-tests/tests/major-upgrade/44-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/44-read-from-primary.yaml deleted file mode 100644 index 7ecb6090a2..0000000000 --- a/e2e-tests/tests/major-upgrade/44-read-from-primary.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 30 -commands: - - script: |- - set -o errexit - set -o xtrace - - source ../../functions - data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)") - - kubectl create configmap -n "${NAMESPACE}" 11-read-from-primary --from-literal=data="${data}" \ No newline at end of file diff --git a/e2e-tests/tests/major-upgrade/45-assert.yaml b/e2e-tests/tests/major-upgrade/45-assert.yaml deleted file mode 100644 index e52b25e2cb..0000000000 --- a/e2e-tests/tests/major-upgrade/45-assert.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 360 -commands: -- script: |- - kubectl -n ${NAMESPACE} get pg,pod,job - sleep 5 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: major-upgrade -spec: - postgresVersion: 17 -status: - pgbouncer: - ready: 3 - size: 3 - postgres: - instances: - - name: instance1 - ready: 3 - size: 3 - ready: 3 - size: 3 - state: ready ---- -kind: Job -apiVersion: batch/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: major-upgrade - postgres-operator.crunchydata.com/pgbackrest: '' - postgres-operator.crunchydata.com/pgbackrest-backup: replica-create - postgres-operator.crunchydata.com/pgbackrest-repo: repo4 - ownerReferences: - - apiVersion: pgv2.percona.com/v2 - kind: PerconaPGBackup - controller: true - blockOwnerDeletion: true -status: - succeeded: 1 diff --git a/e2e-tests/tests/major-upgrade/46-assert.yaml b/e2e-tests/tests/major-upgrade/46-assert.yaml deleted file mode 100644 index 1b95780c8f..0000000000 --- a/e2e-tests/tests/major-upgrade/46-assert.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 660 -commands: -- script: |- - kubectl get postgrescluster major-upgrade \ - -n ${NAMESPACE} \ - -o yaml \ - | yq eval '.status.pgbackrest.repos' - ---- -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: major-upgrade -status: - pgbackrest: - repos: - - bound: true - name: repo1 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/e2e-tests/tests/major-upgrade/47-assert.yaml b/e2e-tests/tests/major-upgrade/47-assert.yaml index 0b0618774c..2b003b69a9 100644 --- a/e2e-tests/tests/major-upgrade/47-assert.yaml +++ b/e2e-tests/tests/major-upgrade/47-assert.yaml @@ -6,7 +6,7 @@ kind: Job apiVersion: batch/v1 metadata: annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-16-to-17 + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-15-to-16 labels: postgres-operator.crunchydata.com/pgbackrest-backup: manual postgres-operator.crunchydata.com/pgbackrest-repo: repo1 @@ -21,7 +21,7 @@ status: apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup metadata: - name: backup-after-16-to-17 + name: backup-after-15-to-16 spec: pgCluster: major-upgrade repoName: repo1 diff --git a/e2e-tests/tests/major-upgrade/47-run-backup.yaml b/e2e-tests/tests/major-upgrade/47-run-backup.yaml index 392bc719ea..3e4fe9c94c 100644 --- a/e2e-tests/tests/major-upgrade/47-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/47-run-backup.yaml @@ -1,7 +1,7 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup metadata: - name: backup-after-16-to-17 + name: backup-after-15-to-16 spec: pgCluster: major-upgrade repoName: repo1 diff --git a/e2e-tests/tests/major-upgrade/48-assert.yaml b/e2e-tests/tests/major-upgrade/48-assert.yaml index 95d57c293f..a870ac130d 100644 --- a/e2e-tests/tests/major-upgrade/48-assert.yaml +++ b/e2e-tests/tests/major-upgrade/48-assert.yaml @@ -1,10 +1,55 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 30 +timeout: 720 +commands: +- script: |- + set -o errexit + + kubectl -n ${NAMESPACE} get pod + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"}) + if [[ "${phase}" != "Running" ]]; then + echo "Waiting for ${pod} to start running" + continue + fi + echo "PostgreSQL logs from ${pod}:" + echo "find /pgdata/pg16/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \ + | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null + done + + sleep 15 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 +- type: pod + selector: "job-name=major-upgrade-pgbackrest-restore" + tail: 300 --- -kind: ConfigMap -apiVersion: v1 +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGRestore metadata: - name: 11-read-from-primary -data: - data: ' 100500' + name: restore-after-15-to-16 +spec: + pgCluster: major-upgrade + repoName: repo1 +status: + state: Succeeded +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: major-upgrade +status: + pgbouncer: + ready: 3 + size: 3 + postgres: + instances: + - name: instance1 + ready: 3 + size: 3 + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/major-upgrade/48-run-restore.yaml b/e2e-tests/tests/major-upgrade/48-run-restore.yaml index d496fe3126..9ca741c97d 100644 --- a/e2e-tests/tests/major-upgrade/48-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/48-run-restore.yaml @@ -1,8 +1,9 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGRestore metadata: - name: restore-after-16-to-17 + name: restore-after-15-to-16 spec: pgCluster: major-upgrade repoName: repo1 - + options: + - --log-level-console=debug diff --git a/e2e-tests/tests/major-upgrade/14-assert.yaml b/e2e-tests/tests/major-upgrade/49-assert.yaml similarity index 55% rename from e2e-tests/tests/major-upgrade/14-assert.yaml rename to e2e-tests/tests/major-upgrade/49-assert.yaml index 7c65443fbe..f13f428d81 100644 --- a/e2e-tests/tests/major-upgrade/14-assert.yaml +++ b/e2e-tests/tests/major-upgrade/49-assert.yaml @@ -5,6 +5,10 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 05-read-from-primary + name: 49-read-from-primary data: - data: ' 100500' + data: |2- + 100500 + 100501 + 100502 + 100503 diff --git a/e2e-tests/tests/major-upgrade/49-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/49-read-from-primary.yaml index 03c7aa852b..0549c2c862 100644 --- a/e2e-tests/tests/major-upgrade/49-read-from-primary.yaml +++ b/e2e-tests/tests/major-upgrade/49-read-from-primary.yaml @@ -9,4 +9,4 @@ commands: source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 11-read-from-primary --from-literal=data="${data}" + kubectl create configmap -n "${NAMESPACE}" 49-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/major-upgrade/45-16-to-17.yaml b/e2e-tests/tests/major-upgrade/50-16-to-17.yaml similarity index 100% rename from e2e-tests/tests/major-upgrade/45-16-to-17.yaml rename to e2e-tests/tests/major-upgrade/50-16-to-17.yaml diff --git a/e2e-tests/tests/major-upgrade/10-assert.yaml b/e2e-tests/tests/major-upgrade/50-assert.yaml similarity index 80% rename from e2e-tests/tests/major-upgrade/10-assert.yaml rename to e2e-tests/tests/major-upgrade/50-assert.yaml index ae8dcb18f8..280e7084f0 100644 --- a/e2e-tests/tests/major-upgrade/10-assert.yaml +++ b/e2e-tests/tests/major-upgrade/50-assert.yaml @@ -5,13 +5,20 @@ commands: - script: |- kubectl -n ${NAMESPACE} get pg,pod,job sleep 5 +collectors: +- type: pod + selector: "postgres-operator.crunchydata.com/data=postgres" + tail: 30 +- type: pod + selector: "postgres-operator.crunchydata.com/pgupgrade=13-to-14" + tail: 200 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: name: major-upgrade spec: - postgresVersion: 13 + postgresVersion: 17 status: pgbouncer: ready: 3 diff --git a/e2e-tests/tests/major-upgrade/51-wait-and-write-data.yaml b/e2e-tests/tests/major-upgrade/51-wait-and-write-data.yaml new file mode 100644 index 0000000000..bcfed0f3d3 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/51-wait-and-write-data.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 90 + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + echo "Checking the status of ${pod}:" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT version()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- psql -c "SELECT timeline_id FROM pg_control_checkpoint()" + kubectl -n ${NAMESPACE} exec -it ${pod} -- pgbackrest --stanza=db --log-level-console=detail check + kubectl -n ${NAMESPACE} exec -it ${pod} -- ls -l /pgdata/pg16_wal + done + + run_psql_local \ + '\c myapp \\\ INSERT INTO myApp (id) VALUES (100504)' \ + "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)" + diff --git a/e2e-tests/tests/major-upgrade/53-recreate-stanza.yaml b/e2e-tests/tests/major-upgrade/53-recreate-stanza.yaml new file mode 100644 index 0000000000..49742c96a3 --- /dev/null +++ b/e2e-tests/tests/major-upgrade/53-recreate-stanza.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 300 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- touch /pgdata/sleep-forever + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stop + done + + pod=$(kubectl get -n ${NAMESPACE} pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | tail -n 1 | awk '{print $1}') + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl pause + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pg_ctl -D /pgdata/pg17 stop + done + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-delete + + for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug start + kubectl -n ${NAMESPACE} exec ${pod} -- rm /pgdata/sleep-forever + done + + kubectl -n ${NAMESPACE} exec ${pod} -- patronictl resume + + # give some time to patroni for starting postgres + sleep 90 + + kubectl -n ${NAMESPACE} exec ${pod} -- pgbackrest --stanza=db --log-level-console=debug stanza-create diff --git a/e2e-tests/tests/major-upgrade/22-assert.yaml b/e2e-tests/tests/major-upgrade/57-assert.yaml similarity index 82% rename from e2e-tests/tests/major-upgrade/22-assert.yaml rename to e2e-tests/tests/major-upgrade/57-assert.yaml index 7a2f6c1c0f..0b0618774c 100644 --- a/e2e-tests/tests/major-upgrade/22-assert.yaml +++ b/e2e-tests/tests/major-upgrade/57-assert.yaml @@ -6,10 +6,10 @@ kind: Job apiVersion: batch/v1 metadata: annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-13-to-14 + postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-16-to-17 labels: postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo3 + postgres-operator.crunchydata.com/pgbackrest-repo: repo1 ownerReferences: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup @@ -21,10 +21,10 @@ status: apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup metadata: - name: backup-after-13-to-14 + name: backup-after-16-to-17 spec: pgCluster: major-upgrade - repoName: repo3 + repoName: repo1 options: - --type=full status: diff --git a/e2e-tests/tests/major-upgrade/42-run-backup.yaml b/e2e-tests/tests/major-upgrade/57-run-backup.yaml similarity index 82% rename from e2e-tests/tests/major-upgrade/42-run-backup.yaml rename to e2e-tests/tests/major-upgrade/57-run-backup.yaml index 3e4fe9c94c..392bc719ea 100644 --- a/e2e-tests/tests/major-upgrade/42-run-backup.yaml +++ b/e2e-tests/tests/major-upgrade/57-run-backup.yaml @@ -1,7 +1,7 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGBackup metadata: - name: backup-after-15-to-16 + name: backup-after-16-to-17 spec: pgCluster: major-upgrade repoName: repo1 diff --git a/e2e-tests/tests/major-upgrade/43-assert.yaml b/e2e-tests/tests/major-upgrade/58-assert.yaml similarity index 91% rename from e2e-tests/tests/major-upgrade/43-assert.yaml rename to e2e-tests/tests/major-upgrade/58-assert.yaml index 3114c23118..a6f87ed937 100644 --- a/e2e-tests/tests/major-upgrade/43-assert.yaml +++ b/e2e-tests/tests/major-upgrade/58-assert.yaml @@ -18,16 +18,19 @@ commands: | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null done - sleep 30 + sleep 15 collectors: - type: pod selector: "postgres-operator.crunchydata.com/data=postgres" tail: 30 +- type: pod + selector: "job-name=major-upgrade-pgbackrest-restore" + tail: 300 --- apiVersion: pgv2.percona.com/v2 kind: PerconaPGRestore metadata: - name: restore-after-15-to-16 + name: restore-after-16-to-17 spec: pgCluster: major-upgrade repoName: repo1 diff --git a/e2e-tests/tests/major-upgrade/43-run-restore.yaml b/e2e-tests/tests/major-upgrade/58-run-restore.yaml similarity index 61% rename from e2e-tests/tests/major-upgrade/43-run-restore.yaml rename to e2e-tests/tests/major-upgrade/58-run-restore.yaml index c86adba0d1..e023013780 100644 --- a/e2e-tests/tests/major-upgrade/43-run-restore.yaml +++ b/e2e-tests/tests/major-upgrade/58-run-restore.yaml @@ -1,8 +1,9 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGRestore metadata: - name: restore-after-15-to-16 + name: restore-after-16-to-17 spec: pgCluster: major-upgrade repoName: repo1 - + options: + - --log-level-console=debug diff --git a/e2e-tests/tests/major-upgrade/44-assert.yaml b/e2e-tests/tests/major-upgrade/59-assert.yaml similarity index 52% rename from e2e-tests/tests/major-upgrade/44-assert.yaml rename to e2e-tests/tests/major-upgrade/59-assert.yaml index 95d57c293f..539ce9a2ad 100644 --- a/e2e-tests/tests/major-upgrade/44-assert.yaml +++ b/e2e-tests/tests/major-upgrade/59-assert.yaml @@ -5,6 +5,11 @@ timeout: 30 kind: ConfigMap apiVersion: v1 metadata: - name: 11-read-from-primary + name: 59-read-from-primary data: - data: ' 100500' + data: |2- + 100500 + 100501 + 100502 + 100503 + 100504 diff --git a/e2e-tests/tests/major-upgrade/34-read-from-primary.yaml b/e2e-tests/tests/major-upgrade/59-read-from-primary.yaml similarity index 76% rename from e2e-tests/tests/major-upgrade/34-read-from-primary.yaml rename to e2e-tests/tests/major-upgrade/59-read-from-primary.yaml index 12ff6f8d79..5f3d5ceba8 100644 --- a/e2e-tests/tests/major-upgrade/34-read-from-primary.yaml +++ b/e2e-tests/tests/major-upgrade/59-read-from-primary.yaml @@ -9,4 +9,4 @@ commands: source ../../functions data=$(run_psql_local '\c myapp \\\ SELECT * from myApp;' "postgres:$(get_psql_user_pass major-upgrade-pguser-postgres)@$(get_psql_user_host major-upgrade-pguser-postgres)") - kubectl create configmap -n "${NAMESPACE}" 09-read-from-primary --from-literal=data="${data}" \ No newline at end of file + kubectl create configmap -n "${NAMESPACE}" 59-read-from-primary --from-literal=data="${data}" diff --git a/e2e-tests/tests/scheduled-backup/07-add-sleep.yaml b/e2e-tests/tests/scheduled-backup/07-add-sleep.yaml index 5fe0203828..e2983b9513 100644 --- a/e2e-tests/tests/scheduled-backup/07-add-sleep.yaml +++ b/e2e-tests/tests/scheduled-backup/07-add-sleep.yaml @@ -5,4 +5,4 @@ commands: set -o errexit set -o xtrace - sleep 2 \ No newline at end of file + sleep 2