chore: Fly refactoring; fixes; extra LSN shipping coverage (#800)

* chore: fly fixes

* chore: configure autoshutdown using remote init payload

* chore: more lsn checkpoint handling

* chore: bump admin-api and admin-mgr

* chore: remove supervisord event listener; use built-in admin-mgr watcher

* chore: rename `db-only` to `base-services`

* chore: logrotate, more tests

Author: pcnc

.github/workflows/dockerhub-release-aio.yml

@@ -13,7 +13,7 @@ on:
       - develop
     types:
       - completed
-    
+
 jobs:
   settings:
     runs-on: ubuntu-latest

.github/workflows/dockerhub-release.yml

@@ -7,7 +7,7 @@ on:
     paths:
       - ".github/workflows/dockerhub-release.yml"
       - "common.vars*"
-
+ 
 jobs:
   settings:
     runs-on: ubuntu-latest
@@ -25,6 +25,7 @@ jobs:
         with:
           cmd: yq 'to_entries | map(select(.value|type == "!!str")) |  map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
 
+
   build_image:
     needs: settings
     strategy:

.github/workflows/testinfra.yml

@@ -14,7 +14,7 @@ jobs:
           - runner: arm-runner
             arch: arm64
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 60
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v3
 
@@ -27,7 +27,17 @@ jobs:
         run: |
           # TODO: use poetry for pkg mgmt
           pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
-          pytest -vv testinfra/test_all_in_one.py
+          
+
+          if ! pytest -vv testinfra/test_all_in_one.py; then
+            # display container logs if the test fails
+            
+            if [ -f testinfra-aio-container-logs.log ]; then
+              echo "AIO container logs:"
+              cat testinfra-aio-container-logs.log
+            fi
+            exit 1
+          fi
 
   test-ami:
     strategy:
@@ -132,9 +142,9 @@ jobs:
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
-          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {}
+          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {}
 
       - name: Cleanup resources on build cancellation
         if: ${{ always() }}
         run: |
-          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {} || true
+          aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true

ansible/vars.yml

@@ -46,8 +46,8 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.58.0
-adminmgr_release: 0.13.1
+adminapi_release: 0.58.1
+adminmgr_release: 0.14.0
 
 # Postgres Extensions
 postgis_release: "3.3.2"

docker/all-in-one/Dockerfile

@@ -1,10 +1,10 @@
-ARG postgres_version=15.1.0.83
+ARG postgres_version=15.1.0.148
 
 ARG pgbouncer_release=1.18.0
 ARG postgrest_release=10.1.2
 ARG gotrue_release=2.47.0
-ARG adminapi_release=0.58.0
-ARG adminmgr_release=0.9.0
+ARG adminapi_release=0.58.1
+ARG adminmgr_release=0.14.0
 ARG vector_release=0.22.3
 ARG postgres_exporter_release=0.9.0
 ARG envoy_release=1.28.0
@@ -144,6 +144,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     sudo \
     vim-tiny \
     less \
+    libnuma1 \
+    logrotate \
     # pg_egress_collect deps
     tcpdump libio-async-perl \
     && rm -rf /var/lib/apt/lists/* /tmp/* \
@@ -197,6 +199,10 @@ COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/bootstrap.sh /e
 COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/base.conf /etc/postgrest/base.conf
 COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/generated.conf /etc/postgrest/generated.conf
 
+# Customizations for logrotate
+COPY docker/all-in-one/etc/logrotate.d/walg.conf /etc/logrotate.d/walg.conf
+COPY docker/all-in-one/etc/logrotate.d/postgresql.conf /etc/logrotate.d/postgresql.conf
+
 # Customizations for gotrue
 COPY docker/all-in-one/etc/gotrue.env /etc/gotrue.env
 
@@ -264,7 +270,7 @@ RUN mkdir -p /var/log/wal-g \
     && chmod ug+s /dist/admin-mgr \
     && touch /etc/wal-g/config.json \
     && chown adminapi:adminapi /etc/wal-g/config.json \
-    && echo '{}' > /etc/wal-g/config.json
+    && echo '{"WALG_S3_PREFIX": "s3://foo/bar/"}' > /etc/wal-g/config.json
 RUN chown -R adminapi:adminapi /etc/adminapi
 
 # Add healthcheck and entrypoint scripts
@@ -275,4 +281,6 @@ COPY docker/all-in-one/init /init
 COPY docker/all-in-one/entrypoint.sh /usr/local/bin/
 COPY docker/all-in-one/postgres-entrypoint.sh /usr/local/bin/
 COPY docker/all-in-one/shutdown.sh /usr/local/bin/supa-shutdown.sh
+COPY docker/all-in-one/run-logrotate.sh /usr/local/bin/run-logrotate.sh
+
 ENTRYPOINT [ "entrypoint.sh" ]

docker/all-in-one/entrypoint.sh

@@ -4,9 +4,11 @@ set -eou pipefail
 PG_CONF=/etc/postgresql/postgresql.conf
 SUPERVISOR_CONF=/etc/supervisor/supervisord.conf
 
-DATA_VOLUME_MOUNTPOINT=${DATA_VOLUME_MOUNTPOINT:-/data}
+export DATA_VOLUME_MOUNTPOINT=${DATA_VOLUME_MOUNTPOINT:-/data}
 export CONFIGURED_FLAG_PATH=${CONFIGURED_FLAG_PATH:-$DATA_VOLUME_MOUNTPOINT/machine.configured}
 
+export MAX_IDLE_TIME_MINUTES=${MAX_IDLE_TIME_MINUTES:-5}
+
 # Ref: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
 function retry {
   # Pass 0 for unlimited retries
@@ -50,14 +52,40 @@ function enable_swap {
   swapon /mnt/swapfile
 }
 
-function create_lsn_checkpoint_file {
-  if [ ! -f "${DATA_VOLUME_MOUNTPOINT}/latest-lsn-checkpoint" ]; then
-    echo "0/0" > "${DATA_VOLUME_MOUNTPOINT}/latest-lsn-checkpoint"
-    chown postgres:postgres "${DATA_VOLUME_MOUNTPOINT}/latest-lsn-checkpoint"
-    chmod 0300 "${DATA_VOLUME_MOUNTPOINT}/latest-lsn-checkpoint"
-  fi
+function push_lsn_checkpoint_file {
+    if [ "${PLATFORM_DEPLOYMENT:-}" != "true" ]; then
+      echo "Skipping push of LSN checkpoint file"
+      return
+    fi
+
+    /usr/bin/admin-mgr lsn-checkpoint-push --immediately || echo "Failed to push LSN checkpoint"
+}
+
+function graceful_shutdown {
+  echo "$(date): Received SIGINT. Shutting down."
+  supervisorctl stop postgresql
+
+  # Postgres ships the latest WAL file using archive_command during shutdown, in a blocking operation
+  # This is to ensure that the WAL file is shipped, just in case
+  sleep 0.2
+  push_lsn_checkpoint_file
+
+  kill -s TERM "$(supervisorctl pid)"
+}
+
+function enable_autoshutdown {
+    sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/supa-shutdown.conf
 }
 
+function enable_lsn_checkpoint_push {
+    sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
+    sed -i "s/autorestart=.*/autorestart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
+}
+
+function disable_fail2ban {
+  sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/fail2ban.conf
+  sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/fail2ban.conf
+}
 
 function setup_postgres {
   tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql.schema.sql
@@ -210,6 +238,8 @@ if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then
   done
 
   chown -R postgres:postgres "${BASE_LOGS_FOLDER}"
+
+  mkdir -p "${DATA_VOLUME_MOUNTPOINT}/etc/logrotate"
 fi
 
 # Process init payload
@@ -229,16 +259,16 @@ find /etc/supervisor/ -type f -exec chmod 0660 {} +
 # Start services in the background
 if [ "${POSTGRES_ONLY:-}" == "true" ]; then
   sed -i "s|    - postgrest|  #  - postgrest|g" /etc/adminapi/adminapi.yaml
-  sed -i "s|files = services/\*.conf db-only/\*.conf|files = db-only/\*.conf|g" $SUPERVISOR_CONF
+  sed -i "s|files = services/\*.conf base-services/\*.conf|files = base-services/\*.conf|g" $SUPERVISOR_CONF
   /init/configure-adminapi.sh
 else
   sed -i "s|  #  - postgrest|    - postgrest|g" /etc/adminapi/adminapi.yaml
-  sed -i "s|files = db-only/\*.conf|files = services/\*.conf db-only/\*.conf|g" $SUPERVISOR_CONF
+  sed -i "s|files = base-services/\*.conf|files = services/\*.conf base-services/\*.conf|g" $SUPERVISOR_CONF
   configure_services
 fi
 
 if [ "${AUTOSHUTDOWN_ENABLED:-}" == "true" ]; then
-  sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/db-only/supa-shutdown.conf
+  enable_autoshutdown
 fi
 
 if [ "${ENVOY_ENABLED:-}" == "true" ]; then
@@ -248,8 +278,7 @@ if [ "${ENVOY_ENABLED:-}" == "true" ]; then
 fi
 
 if [ "${FAIL2BAN_DISABLED:-}" == "true" ]; then
-  sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/fail2ban.conf
-  sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/fail2ban.conf
+  disable_fail2ban
 fi
 
 if [ "${GOTRUE_DISABLED:-}" == "true" ]; then
@@ -258,9 +287,14 @@ if [ "${GOTRUE_DISABLED:-}" == "true" ]; then
 fi
 
 if [ "${PLATFORM_DEPLOYMENT:-}" == "true" ]; then
-  enable_swap
-  create_lsn_checkpoint_file
+  if [ "${SWAP_DISABLED:-}" != "true" ]; then
+    enable_swap
+  fi
+  enable_lsn_checkpoint_push
+
+  trap graceful_shutdown SIGINT
 fi
 
 touch "$CONFIGURED_FLAG_PATH"
 start_supervisor
+push_lsn_checkpoint_file

docker/all-in-one/etc/adminapi/adminapi.yaml

@@ -9,6 +9,7 @@ metric_collectors:
   - loadavg
   - cpu
   - diskstats
+  - vmstat
 node_exporter_additional_args:
   - "--collector.filesystem.ignored-mount-points=^/(boot|sys|dev|run).*"
   - "--collector.netdev.device-exclude=lo"

docker/all-in-one/etc/logrotate.d/postgresql.conf

@@ -0,0 +1,11 @@
+/var/log/postgresql/postgresql.csv {
+    size 50M
+    rotate 4
+    compress
+    delaycompress
+    notifempty
+    missingok
+    postrotate
+        sudo -u postgres /usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data logrotate
+    endscript
+}

docker/all-in-one/etc/logrotate.d/walg.conf

@@ -0,0 +1,9 @@
+/var/log/wal-g/*.log {
+    size 50M
+    rotate 3  
+    copytruncate
+    delaycompress
+    compress
+    notifempty
+    missingok
+}

docker/all-in-one/etc/supervisor/db-only/adminapi.conf renamed to docker/all-in-one/etc/supervisor/base-services/adminapi.conf

@@ -0,0 +1,10 @@
+[program:logrotate]
+command=/usr/local/bin/run-logrotate.sh
+autostart=true
+autorestart=true
+user=root
+stdout_logfile=/var/log/services/logrotate.log
+redirect_stderr=true
+stdout_logfile_maxbytes=10MB
+priority=50
+environment=DATA_VOLUME_MOUNTPOINT="%(ENV_DATA_VOLUME_MOUNTPOINT)s"

docker/all-in-one/etc/supervisor/base-services/logrotate.conf

@@ -0,0 +1,9 @@
+[program:lsn-checkpoint-push]
+command=/usr/bin/admin-mgr lsn-checkpoint-push --watch
+user=root
+autorestart=false
+autostart=false
+stdout_logfile=/var/log/services/lsn-push.log
+redirect_stderr=true
+stdout_logfile_maxbytes=10MB
+priority=50

docker/all-in-one/etc/supervisor/base-services/lsn-checkpoint-push.conf

@@ -7,3 +7,4 @@ stdout_logfile=/var/log/services/supa-shutdown.log
 redirect_stderr=true
 stdout_logfile_maxbytes=10MB
 priority=50
+environment=MAX_IDLE_TIME_MINUTES="%(ENV_MAX_IDLE_TIME_MINUTES)s"

docker/all-in-one/etc/supervisor/db-only/pg_egress_collect.conf renamed to docker/all-in-one/etc/supervisor/base-services/pg_egress_collect.conf

@@ -167,4 +167,4 @@ serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
 ; include files themselves.
 
 [include]
-files = db-only/*.conf
+files = base-services/*.conf

docker/all-in-one/etc/supervisor/db-only/postgresql.conf renamed to docker/all-in-one/etc/supervisor/base-services/postgresql.conf

@@ -1,4 +1,8 @@
 #!/bin/bash
 set -eou pipefail
 
+touch "/var/log/wal-g/pitr.log"
+chown postgres:postgres "/var/log/wal-g/pitr.log"
+chmod 0666 "/var/log/wal-g/pitr.log"
+
 /usr/local/bin/configure-shim.sh /dist/admin-mgr /usr/bin/admin-mgr

docker/all-in-one/etc/supervisor/db-only/supa-shutdown.conf renamed to docker/all-in-one/etc/supervisor/base-services/supa-shutdown.conf

@@ -8,21 +8,6 @@ ADMINAPI_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/adminapi"
 
 /usr/local/bin/configure-shim.sh /dist/supabase-admin-api /opt/supabase-admin-api
 
-if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then
-  mkdir -p "${ADMINAPI_CUSTOM_DIR}"
-  if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then
-    echo "Copying existing custom adminapi config from /etc/adminapi to ${ADMINAPI_CUSTOM_DIR}"
-    cp -R "/etc/adminapi/." "${ADMINAPI_CUSTOM_DIR}/"
-  fi
-
-  rm -rf "/etc/adminapi"
-  ln -s "${ADMINAPI_CUSTOM_DIR}" "/etc/adminapi"
-  chown -R adminapi:adminapi "/etc/adminapi"
-
-  chown -R adminapi:adminapi "${ADMINAPI_CUSTOM_DIR}"
-  chmod g+rx "${ADMINAPI_CUSTOM_DIR}"
-fi
-
 if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then
   echo "init adminapi payload"
   tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/adminapi/adminapi.yaml
@@ -53,3 +38,18 @@ chmod -R 0775 /etc/postgresql-custom
 
 # Update api port
 sed -i "s|^port: .*$|port: ${ADMIN_API_PORT:-8085}|g" $ADMIN_API_CONF
+
+if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then
+  mkdir -p "${ADMINAPI_CUSTOM_DIR}"
+  if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then
+    echo "Copying existing custom adminapi config from /etc/adminapi to ${ADMINAPI_CUSTOM_DIR}"
+    cp -R "/etc/adminapi/." "${ADMINAPI_CUSTOM_DIR}/"
+  fi
+
+  rm -rf "/etc/adminapi"
+  ln -s "${ADMINAPI_CUSTOM_DIR}" "/etc/adminapi"
+  chown -R adminapi:adminapi "/etc/adminapi"
+
+  chown -R adminapi:adminapi "${ADMINAPI_CUSTOM_DIR}"
+  chmod g+rx "${ADMINAPI_CUSTOM_DIR}"
+fi

docker/all-in-one/etc/supervisor/supervisord.conf

@@ -3,7 +3,7 @@ set -eou pipefail
 
 mkdir -p /etc/supa-shutdown
 
-AUTOSHUTDOWN_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/supa-shutdown"
+AUTOSHUTDOWN_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/supa-shutdown"
 if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then
   mkdir -p "${AUTOSHUTDOWN_CUSTOM_DIR}"
 

docker/all-in-one/init/configure-admin-mgr.sh

@@ -3,15 +3,15 @@ set -eou pipefail
 
 touch /var/log/services/gotrue.log
 
-/usr/local/bin/configure-shim.sh /dist/gotrue /opt/gotrue/gotrue
+GOTRUE_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/gotrue"
+GOTRUE_CUSTOM_CONFIG_FILE_PATH="${DATA_VOLUME_MOUNTPOINT}/etc/gotrue/gotrue.env"
 
-sed -i "s|api_external_url|${API_EXTERNAL_URL:-http://localhost}|g" /etc/gotrue.env
-sed -i "s|gotrue_api_host|${GOTRUE_API_HOST:-0.0.0.0}|g" /etc/gotrue.env
-sed -i "s|gotrue_site_url|$GOTRUE_SITE_URL|g" /etc/gotrue.env
-sed -i "s|gotrue_jwt_secret|$JWT_SECRET|g" /etc/gotrue.env
+/usr/local/bin/configure-shim.sh /dist/gotrue /opt/gotrue/gotrue
 
 if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then
-  GOTRUE_CUSTOM_CONFIG_FILE_PATH="${DATA_VOLUME_MOUNTPOINT}/etc/gotrue.env"
+  mkdir -p "${GOTRUE_CUSTOM_DIR}"
+  chown adminapi:adminapi "${GOTRUE_CUSTOM_DIR}"
+
   if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then
     echo "Copying existing GoTrue config from /etc/gotrue.env to ${GOTRUE_CUSTOM_CONFIG_FILE_PATH}"
     cp "/etc/gotrue.env" "${GOTRUE_CUSTOM_CONFIG_FILE_PATH}"
@@ -27,6 +27,11 @@ fi
 
 if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then
   echo "init gotrue payload"
-  tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/gotrue.env
+  tar -h --overwrite -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/gotrue.env
   chown -R adminapi:adminapi /etc/gotrue.env
+else
+  sed -i "s|api_external_url|${API_EXTERNAL_URL:-http://localhost}|g" /etc/gotrue.env
+  sed -i "s|gotrue_api_host|${GOTRUE_API_HOST:-0.0.0.0}|g" /etc/gotrue.env
+  sed -i "s|gotrue_site_url|$GOTRUE_SITE_URL|g" /etc/gotrue.env
+  sed -i "s|gotrue_jwt_secret|$JWT_SECRET|g" /etc/gotrue.env
 fi