chore: build pg15 with docker

Author: sweatybridge

.github/workflows/ami-release.yml

@@ -56,6 +56,25 @@ jobs:
             tar xvf "$layer" -C ansible/files/extensions --strip-components 1
           done
 
+      - name: Build Postgres deb
+        uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          file: docker/Dockerfile
+          target: pg-deb
+          tags: supabase/postgres:deb
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Extract Postgres deb
+        run: |
+          mkdir -p /tmp/build ansible/files/postgres
+          docker save supabase/postgres:deb | tar xv -C /tmp/build
+          for layer in /tmp/build/*/layer.tar; do
+            tar xvf "$layer" -C ansible/files/postgres --strip-components 1
+          done
+
       - name: Build AMI
         run: |
           GIT_SHA=${{github.sha}}

ansible/tasks/setup-postgres.yml

@@ -1,70 +1,13 @@
-# Downloading dependencies
-- name: GPG dependencies
-  apt:
-    name: gnupg
-
-- name: Add Postgres GPG key
-  apt_key:
-    keyserver: keyserver.ubuntu.com
-    id: B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
-
-- name: Add Postgres PPA
-  apt_repository:
-    repo: "deb https://apt-archive.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg-archive main"
-    state: present
+- name: Postgres - copy package
+  copy:
+    src: files/postgres/
+    dest: /tmp/build/
 
-# let's build binaries from their published source packages
-- name: Add Postgres PPA - source
+- name: Postgres - add PPA
   apt_repository:
-    repo: "deb-src https://apt-archive.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg-archive main"
+    repo: "deb [ trusted=yes ] file:///tmp/build ./"
     state: present
 
-- name: Create temporary build directory
-  tempfile:
-    state: directory
-  register: pg_build_dir
-
-- name: Setting mcpu (arm)
-  set_fact:
-    mcpu: "neoverse-n1"
-  when: platform == "arm64"
-
-- name: Postgres - build
-  shell: |
-    set -e
-    export PYTHONDONTWRITEBYTECODE=1
-    savedAptMark="$(apt-mark showmanual)"
-    cd "{{ pg_build_dir.path }}"
-
-    # create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be)
-    apt-get update
-    apt-get install -y --no-install-recommends dpkg-dev
-    echo "deb [ trusted=yes ] file://{{ pg_build_dir.path }} ./" > /etc/apt/sources.list.d/temp.list
-    _update_repo() {
-      dpkg-scanpackages . > Packages
-      apt-get -o Acquire::GzipIndexes=false update
-    }
-    _update_repo
-
-    # build .deb files from upstream's source packages (which are verified by apt-get)
-    export DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)"
-    export DEB_CPPFLAGS_APPEND="-mcpu={{ mcpu }} -fsigned-char"
-
-    # we have to build postgresql-common first because postgresql-15 shares "debian/rules" logic with it: https://salsa.debian.org/postgresql/postgresql/-/commit/99f44476e258cae6bf9e919219fa2c5414fa2876
-    # (and it "Depends: pgdg-keyring")
-    apt-get build-dep -y postgresql-common pgdg-keyring
-    apt-get source --compile postgresql-common pgdg-keyring
-    _update_repo
-    apt-get build-dep -y "postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1"
-    apt-get source --compile "postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1"
-
-    # we don't remove APT lists here because they get re-downloaded and removed later
-    # reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies
-    # (which is done after we install the built packages so we don't have to redownload any overlapping dependencies)
-    apt-mark showmanual | xargs apt-mark auto > /dev/null
-    apt-mark manual $savedAptMark
-    _update_repo
-
 - name: Postgres - install commons
   apt:
     name: postgresql-common
@@ -79,17 +22,15 @@
     name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1
     install_recommends: no
 
-# if we have leftovers from building, let's purge them (including extra, unnecessary build deps)
-- name: Remove build dependencies
-  shell: |
-    set -e
-    rm -rf /var/lib/apt/lists/*
-    apt-get purge -y --auto-remove
-    rm -rf "{{ pg_build_dir.path }}" /etc/apt/sources.list.d/temp.list
-    find /usr -name '*.pyc' -type f -exec bash -c 'for pyc; do dpkg -S "$pyc" &> /dev/null || rm -vf "$pyc"; done' -- '{}' +
-
-- name: Hold postgres {{ postgresql_release }} from apt upgrade
-  shell: apt-mark hold postgresql-{{ postgresql_major }}
+- name: Postgres - remove PPA
+  apt_repository:
+    repo: "deb [ trusted=yes ] file:///tmp/build ./"
+    state: absent
+
+- name: Postgres - cleanup package
+  file:
+    path: /tmp/build
+    state: absent
 
 - name: Create symlink to /usr/lib/postgresql/bin
   shell:

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.0.89-rc0"
+postgres-version = "15.1.0.89-rc1"

docker/Dockerfile

@@ -0,0 +1,71 @@
+ARG ubuntu_release=focal
+FROM ubuntu:${ubuntu_release} as base
+
+ARG ubuntu_release
+ARG postgresql_major=15
+ARG postgresql_release=${postgresql_major}.1
+
+FROM base as pg-source
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gnupg \
+    dpkg-dev \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Add Postgres PPA
+ARG postgresql_gpg_key=B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys "${postgresql_gpg_key}" && \
+    echo "deb https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list && \
+    echo "deb-src https://apt-archive.postgresql.org/pub/repos/apt ${ubuntu_release}-pgdg-archive main" > /etc/apt/sources.list.d/pgdg.list
+
+# Create local PPA
+WORKDIR /tmp/build
+RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \
+    dpkg-scanpackages . > Packages && \
+    apt-get -o Acquire::GzipIndexes=false update
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)"
+
+# Configure processor optimised build
+ARG CPPFLAGS="-mcpu=neoverse-n1"
+ENV DEB_CPPFLAGS_APPEND="${CPPFLAGS} -fsigned-char"
+
+RUN apt-get build-dep -y postgresql-common pgdg-keyring && \
+    apt-get source --compile postgresql-common pgdg-keyring && \
+    dpkg-scanpackages . > Packages && \
+    apt-get -o Acquire::GzipIndexes=false update
+
+RUN apt-get build-dep -y "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+    apt-get source --compile "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+    dpkg-scanpackages . > Packages && \
+    apt-get -o Acquire::GzipIndexes=false update
+
+# Remove source directories
+RUN rm -rf /tmp/build/*/
+
+FROM base as pg
+
+# Inherit args from base stage
+ARG postgresql_major
+ARG postgresql_release
+
+COPY --from=pg-source /tmp/build /tmp/build
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN echo "deb [ trusted=yes ] file:///tmp/build ./" > /etc/apt/sources.list.d/temp.list && \
+    apt-get -o Acquire::GzipIndexes=false update && \
+    apt-get install -y --no-install-recommends postgresql-common && \
+    sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf && \
+    apt-get install -y --no-install-recommends "postgresql-${postgresql_major}=${postgresql_release}-1.pgdg20.04+1" && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/build /etc/apt/sources.list.d/temp.list
+
+ENV PATH $PATH:/usr/lib/postgresql/${postgresql_major}/bin
+
+FROM scratch as pg-deb
+
+COPY --from=pg-source /tmp/build /tmp