feat: support a pg 14 version of postgres bundle

.github/workflows/ami-release-nix-single.yml

@@ -28,6 +28,11 @@ jobs:
         with:
           ref: ${{ github.event.inputs.branch }}
 
+      - name: Get current branch SHA
+        id: get_sha
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
       - uses: DeterminateSystems/nix-installer-action@main
 
       - name: Set PostgreSQL version environment variable
@@ -46,15 +51,15 @@ jobs:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
         run: |
           packer init amazon-arm64-nix.pkr.hcl
-          GIT_SHA=${{github.sha}}
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
           packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
 
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
         run: |
           packer init stage2-nix-psql.pkr.hcl
-          GIT_SHA=${{github.sha}}
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
           POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
           packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
 
@@ -66,7 +71,7 @@ jobs:
 
       - name: Create nix flake revision tarball
         run: |
-          GIT_SHA=${{github.sha}}
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
           MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
@@ -116,7 +121,7 @@ jobs:
         with:
           name: ${{ steps.process_release_version.outputs.version }}
           tag_name: ${{ steps.process_release_version.outputs.version }}
-          target_commitish: ${{github.sha}}
+          target_commitish: ${{ steps.get_sha.outputs.sha }}
 
       - name: Slack Notification on Failure
         if: ${{ failure() }}
@@ -138,3 +143,136 @@ jobs:
         run: |
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids 
 
+name: Release Single AMI Nix
+
+on:
+  workflow_dispatch:
+    inputs:
+      postgres_version:
+        description: 'PostgreSQL major version to build (e.g. 15)'
+        required: true
+        type: string
+      branch:
+        description: 'Branch to run the workflow from'
+        required: true
+        type: string
+        default: 'main'
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: arm-runner
+    timeout-minutes: 150
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.branch }}
+
+      - name: Get current branch SHA
+        id: get_sha
+        run: |
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL version environment variable
+        run: echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
+
+      - name: Generate common-nix.vars.pkr.hcl
+        run: |
+          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
+          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
+          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+          # Ensure there's a newline at the end of the file
+          echo "" >> common-nix.vars.pkr.hcl
+      - name: Build AMI stage 1
+        env:
+          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
+        run: |
+          packer init amazon-arm64-nix.pkr.hcl
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
+          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
+      - name: Build AMI stage 2
+        env:
+          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
+        run: |
+          packer init stage2-nix-psql.pkr.hcl
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
+          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+      - name: Create nix flake revision tarball
+        run: |
+          GIT_SHA=${{ steps.get_sha.outputs.sha }}
+          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
+          echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
+          tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" .
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload software manifest to s3 staging
+        run: |
+          cd ansible
+          ansible-playbook -i localhost \
+            -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
+            -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
+            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            manifest-playbook.yml
+      - name: Upload nix flake revision to s3 staging
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+      - name: configure aws credentials - prod
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload software manifest to s3 prod
+        run: |
+          cd ansible
+          ansible-playbook -i localhost \
+            -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
+            -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
+            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            manifest-playbook.yml
+
+      - name: Upload nix flake revision to s3 prod
+        run: |
+          aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ steps.process_release_version.outputs.version }}
+          tag_name: ${{ steps.process_release_version.outputs.version }}
+          target_commitish: ${{ steps.get_sha.outputs.sha }}
+
+      - name: Slack Notification on Failure
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          SLACK_USERNAME: 'gha-failures-notifier'
+          SLACK_COLOR: 'danger'
+          SLACK_MESSAGE: 'Building Postgres AMI failed'
+          SLACK_FOOTER: ''
+
+      - name: Cleanup resources after build
+        if: ${{ always() }}
+        run: |
+          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
+      - name: Cleanup resources on build cancellation
+        if: ${{ cancelled() }}
+        run: |
+          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids 

.github/workflows/test.yml

@@ -81,6 +81,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
       - name: verify schema.sql is committed
         run: |
+          nix-collect-garbage -d
           nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }}
           if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
             echo "Detected changes in schema.sql:"

ansible/vars.yml

@@ -9,9 +9,15 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.087-orioledb"
-  postgres17: "17.4.1.037"
+  postgresorioledb-17: "17.0.1.088-orioledb"
+  postgres17: "17.4.1.038"
   postgres15: "15.8.1.094"
+  postgres14: "14.18.1.001-test-3"
+
+# Note, postgres14 is not built on every release, but is available in the nix cache. 
+# postgres14 is included here to support upgrading from postgres14 to newer versions.
+# an image can be created with a special workflow that builds postgres14 image. 
+# Supabase will not create and release postgres14 images other than as a utility for internal use.
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

docker/nix/build_nix.sh

@@ -14,6 +14,7 @@ nix build .#checks.$SYSTEM.psql_17 -L --no-link
 nix build .#psql_15/bin -o psql_15 -L
 nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L
 nix build .#psql_17/bin -o psql_17 -L
+nix build .#psql_14/bin -o psql_14 -L
 nix build .#wal-g-2 -o wal-g-2 -L
 nix build .#wal-g-3 -o wal-g-3 -L
 
@@ -23,6 +24,7 @@ nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_17
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_14
 
 if [ "$SYSTEM" = "aarch64-linux" ]; then
     nix build .#postgresql_15_debug -o ./postgresql_15_debug
@@ -31,6 +33,10 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
     nix build .#postgresql_orioledb-17_src -o ./postgresql_orioledb-17_src
     nix build .#postgresql_17_debug -o ./postgresql_17_debug
     nix build .#postgresql_17_src -o ./postgresql_17_src
+    nix build .#postgresql_14_debug -o ./postgresql_14_debug
+    nix build .#postgresql_14_src -o ./postgresql_14_src
+    nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_14_debug-debug
+    nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_14_src
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_debug-debug
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_15_src
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug