chore: fix binary collection; install .deb pg binaries for u18 (#681)

pcnc · web-flow · commit b3e089c245e6 · 2023-06-27T20:31:56.000+03:00
* chore: fix binary collection; install .deb pg binaries on u18

* chore: ubuntu 18 postgres .deb install compat

* chore: bump version

* chore: Install ubuntu 18 deps

* chore: additional handling

* chore: bump version

* chore: more library locations

* chore: bump postgres version
diff --git a/.github/workflows/ami-build-ubuntu-18.yml b/.github/workflows/ami-build-ubuntu-18.yml
@@ -18,8 +18,16 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
-    runs-on: [self-hosted, X64]
+  build:     
+    strategy:
+      matrix:
+        include:
+        - runner: arm-runner
+          arch: arm64
+          ubuntu_release: bionic
+          ubuntu_version: 18.04
+          mcpu: neoverse-n1
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 150
     permissions:
       contents: write
@@ -28,8 +36,62 @@ jobs:
 
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
+      - id: args
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq 'to_entries | map(select(.value|type == "!!str")) |  map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders          
+      - uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          build-args: |
+            ${{ steps.args.outputs.result }}
+          target: extensions
+          tags: supabase/postgres:extensions
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Extract built packages
+        run: |
+          mkdir -p /tmp/extensions ansible/files/extensions
+          docker save supabase/postgres:extensions | tar xv -C /tmp/extensions
+          for layer in /tmp/extensions/*/layer.tar; do
+            tar xvf "$layer" -C ansible/files/extensions --strip-components 1
+          done
+
+      - id: version
+        run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT"
+      - name: Build Postgres deb
+        uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          file: docker/Dockerfile
+          target: pg-deb
+          build-args: |
+            ubuntu_release=${{ matrix.ubuntu_release }}
+            ubuntu_release_no=${{ matrix.ubuntu_version }}
+            postgresql_major=${{ steps.version.outputs.postgresql_major }}
+            postgresql_release=${{ steps.version.outputs.postgresql_release }}
+            CPPFLAGS=-mcpu=${{ matrix.mcpu }}
+          tags: supabase/postgres:deb-u18
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Extract Postgres deb
+        run: |
+          mkdir -p /tmp/build ansible/files/postgres
+          docker save supabase/postgres:deb-u18 | tar xv -C /tmp/build
+          for layer in /tmp/build/*/layer.tar; do
+            tar xvf "$layer" -C ansible/files/postgres --strip-components 1
+          done
       - name: Build AMI
         run: |
           GIT_SHA=${{github.sha}}
diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml
@@ -16,7 +16,8 @@ jobs:
         include:
           - runner: arm-runner
             arch: arm64
-            ubuntu: focal
+            ubuntu_release: focal
+            ubuntu_version: 20.04
             mcpu: neoverse-n1
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 150
@@ -66,7 +67,8 @@ jobs:
           file: docker/Dockerfile
           target: pg-deb
           build-args: |
-            ubuntu_release=${{ matrix.ubuntu }}
+            ubuntu_release=${{ matrix.ubuntu_release }}
+            ubuntu_release_no=${{ matrix.ubuntu_version }}
             postgresql_major=${{ steps.version.outputs.postgresql_major }}
             postgresql_release=${{ steps.version.outputs.postgresql_release }}
             CPPFLAGS=-mcpu=${{ matrix.mcpu }}
diff --git a/Dockerfile b/Dockerfile
@@ -24,7 +24,7 @@ ARG rum_release=1.3.13
 ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
 ARG libsodium_release=1.0.18
 ARG pgsodium_release=3.1.6
-ARG pg_graphql_release=1.1.0
+ARG pg_graphql_release=1.2.2
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
 ARG vault_release=0.2.8
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_complete.sh
@@ -7,8 +7,9 @@
 
 set -eEuo pipefail
 
+SCRIPT_DIR=$(dirname -- "$0";)
 # shellcheck disable=SC1091
-source ./pg_upgrade_common.sh
+source "$SCRIPT_DIR/pg_upgrade_common.sh"
 
 LOG_FILE="/tmp/pg-upgrade-complete.log"
 
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_initiate.sh
@@ -27,8 +27,9 @@ PG13_EXTENSIONS_TO_DISABLE=(
 
 set -eEuo pipefail
 
+SCRIPT_DIR=$(dirname -- "$0";)
 # shellcheck disable=SC1091
-source ./pg_upgrade_common.sh
+source "$SCRIPT_DIR/pg_upgrade_common.sh"
 
 LOG_FILE="/var/log/pg-upgrade-initiate.log"
 
@@ -68,13 +69,17 @@ cleanup() {
         ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true
     fi
 
-    if [ -L /var/lib/postgresql ]; then
-        rm /var/lib/postgresql
-        mv /var/lib/postgresql.bak /var/lib/postgresql
+    if [ -L "/usr/share/postgresql/${PGVERSION}" ]; then
+        rm "/usr/share/postgresql/${PGVERSION}"
+
+        if [ -f "/usr/share/postgresql/${PGVERSION}.bak" ]; then
+            mv "/usr/share/postgresql/${PGVERSION}.bak" "/usr/share/postgresql/${PGVERSION}"
+        fi
     fi
 
-    if [ -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then
+    if [ -f "/usr/lib/postgresql/lib/aarch64/libpq.so.5.bak" ]; then
         rm /usr/lib/postgresql/lib/aarch64/libpq.so.5
+        mv /usr/lib/postgresql/lib/aarch64/libpq.so.5.bak /usr/lib/postgresql/lib/aarch64/libpq.so.5
     fi
 
     if [ "$IS_DRY_RUN" = false ]; then
@@ -169,10 +174,11 @@ function initiate_upgrade {
     chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION"
 
     # Make latest libpq available to pg_upgrade
-    mkdir -p /usr/lib/postgresql/lib/aarch64
-    if [ ! -L /usr/lib/postgresql/lib/aarch64/libpq.so.5 ]; then
-        ln -s "$PGLIBNEW/libpq.so.5" /usr/lib/postgresql/lib/aarch64/libpq.so.5
+    mkdir -p /usr/lib/aarch64-linux-gnu
+    if [ -f "/usr/lib/aarch64-linux-gnu/libpq.so.5" ]; then
+        mv /usr/lib/aarch64-linux-gnu/libpq.so.5 /usr/lib/aarch64-linux-gnu/libpq.so.5.bak
     fi
+    ln -s "$PG_UPGRADE_BIN_DIR/libpq.so.5" /usr/lib/aarch64-linux-gnu/libpq.so.5
 
     # upgrade job outputs a log in the cwd; needs write permissions
     mkdir -p /tmp/pg_upgrade/
@@ -221,6 +227,14 @@ function initiate_upgrade {
     echo "7. Granting SUPERUSER to postgres user"
     run_sql -c "ALTER USER postgres WITH SUPERUSER;"
 
+    if [ -d "/usr/share/postgresql/${PGVERSION}" ]; then
+        mv "/usr/share/postgresql/${PGVERSION}" "/usr/share/postgresql/${PGVERSION}.bak"
+    fi
+    ln -s "$PGSHARENEW" "/usr/share/postgresql/${PGVERSION}"
+
+    cp --remove-destination "$PGLIBNEW"/*.control "$PGSHARENEW/extension/"
+    cp --remove-destination "$PGLIBNEW"/*.sql "$PGSHARENEW/extension/"
+
     echo "8. Creating new data directory, initializing database"
     chown -R postgres:postgres "$MOUNT_POINT/"
     rm -rf "${PGDATANEW:?}/"
@@ -243,20 +257,6 @@ EOF
     if [ "$IS_DRY_RUN" = true ]; then
         UPGRADE_COMMAND="$UPGRADE_COMMAND --check"
     else 
-        mv /var/lib/postgresql /var/lib/postgresql.bak
-        ln -s "$PGSHARENEW" /var/lib/postgresql
-
-        if [ ! -L /var/lib/postgresql.bak/data ]; then
-            if [ -L /var/lib/postgresql/data ]; then
-                rm /var/lib/postgresql/data
-            fi
-            ln -s /var/lib/postgresql.bak/data /var/lib/postgresql/data
-        fi
-        
-        if [ ! -L /var/lib/postgresql/data ]; then
-            ln -s /data/pgdata /var/lib/postgresql/data
-        fi
-
         echo "9. Stopping postgres; running pg_upgrade"
         systemctl stop postgresql
     fi
diff --git a/ansible/tasks/internal/collect-pg-binaries.yml b/ansible/tasks/internal/collect-pg-binaries.yml
@@ -6,16 +6,34 @@
 - name: Collect Postgres binaries - collect binaries and libraries
   copy:
     remote_src: yes
-    src: /usr/lib/postgresql/{{ item }}/
+    src: /usr/lib/postgresql/{{ postgresql_major }}/{{ item }}/
     dest: /tmp/pg_binaries/{{ postgresql_major }}/{{ item }}/
   with_items:
     - bin
     - lib
 
+- name: Collect Postgres libraries - collect libraries which are in /usr/lib/postgresql/lib/
+  copy:
+    remote_src: yes
+    src: /usr/lib/postgresql/lib/
+    dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/
+
+- name: Collect Postgres libraries - collect libraries which are in /var/lib/postgresql/extension/
+  copy:
+    remote_src: yes
+    src: /var/lib/postgresql/extension/
+    dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/
+
+- name: Collect Postgres libraries - collect latest libpq
+  copy:
+    remote_src: yes
+    src: /usr/lib/aarch64-linux-gnu/libpq.so.5
+    dest: /tmp/pg_binaries/{{ postgresql_major }}/libpq.so.5
+
 - name: Collect Postgres binaries - collect shared files
   copy:
     remote_src: yes
-    src: /var/lib/postgresql/
+    src: /usr/share/postgresql/{{ postgresql_major }}/
     dest: /tmp/pg_binaries/{{ postgresql_major }}/share/
 
 - name: Collect Postgres binaries - create tarfile
diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml
@@ -15,7 +15,7 @@
       - clang-12
     update_cache: yes
     cache_valid_time: 3600
-  when: ansible_distribution_version != "18.04"
+  when: ansible_distribution_version == "18.04"
 
 - name: supautils - download latest release
   get_url:
diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl
@@ -1 +1 @@
-postgres-version = "15.1.0.98"
+postgres-version = "15.1.0.99"
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -2,6 +2,7 @@ ARG ubuntu_release=focal
 FROM ubuntu:${ubuntu_release} as base
 
 ARG ubuntu_release
+ARG ubuntu_release_no
 ARG postgresql_major=15
 ARG postgresql_release=${postgresql_major}.1
 
@@ -39,8 +40,8 @@ RUN apt-get build-dep -y postgresql-common pgdg-keyring && \
     dpkg-scanpackages . > Packages && \
     apt-get -o Acquire::GzipIndexes=false update
 
-RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
-    apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \
+    apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \
     dpkg-scanpackages . > Packages && \
     apt-get -o Acquire::GzipIndexes=false update
 
@@ -60,7 +61,7 @@ RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/te
     apt-get -o Acquire::GzipIndexes=false update && \
     apt-get install -y --no-install-recommends postgresql-common && \
     sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf && \
-    apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+    apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg${ubuntu_release_no}+1" && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /tmp/build /etc/apt/sources.list.d/temp.list
 

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-postgres-version = "15.1.0.98"`
	`1`	`+postgres-version = "15.1.0.99"`