Skip to content
AutoGPT Platform - Backend CI

Feat: Add HeyGen Integration #23319

Sign in to view logs

Feat: Add HeyGen Integration

Feat: Add HeyGen Integration #23319

Workflow file for this run

.github/workflows/platform-backend-ci.yml at 614f523
name: AutoGPT Platform - Backend CI
on:
push:
branches: [master, dev, ci-test*]
paths:
- ".github/workflows/platform-backend-ci.yml"
- ".github/workflows/scripts/get_package_version_from_lockfile.py"
- "autogpt_platform/backend/**"
- "autogpt_platform/autogpt_libs/**"
pull_request:
branches: [master, dev, release-*]
paths:
- ".github/workflows/platform-backend-ci.yml"
- ".github/workflows/scripts/get_package_version_from_lockfile.py"
- "autogpt_platform/backend/**"
- "autogpt_platform/autogpt_libs/**"
merge_group:
# On-demand run of the full backend test + coverage suite for any branch.
# Useful when a branch's changes don't match the paths filter above (so the
# automatic push/pull_request runs are skipped) but you still want to run the
# suite and produce a fresh coverage upload for that branch's HEAD commit.
# gh workflow run platform-backend-ci.yml --ref <branch>
# Pass pr_number to attach the coverage upload to an open PR (refreshes that
# PR's codecov/project/platform-backend status):
# gh workflow run platform-backend-ci.yml --ref <branch> -f pr_number=<PR#>
workflow_dispatch:
inputs:
pr_number:
description: "Open PR number to attach this coverage upload to (refreshes that PR's codecov status). Leave blank for a plain branch run."
required: false
type: string
concurrency:
group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
defaults:
run:
shell: bash
working-directory: autogpt_platform/backend
jobs:
lint:
permissions:
contents: read
timeout-minutes: 10
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Set up Python dependency cache
uses: actions/cache@v5
with:
path: ~/.cache/pypoetry
key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
- name: Install Poetry
run: |
HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
echo "Using Poetry version ${HEAD_POETRY_VERSION}"
curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
- name: Install Python dependencies
run: poetry install
- name: Run Linters
run: poetry run lint --skip-pyright
env:
CI: true
PLAIN_OUTPUT: True
# autogpt_libs is a standalone package with its own env; the backend `test`
# job (scoped to autogpt_platform/backend) never collects its tests, so run
# them here. Pure unit tests — no database or external services needed.
autogpt-libs-test:
permissions:
contents: read
timeout-minutes: 10
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: autogpt_platform/autogpt_libs
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Set up Python dependency cache
uses: actions/cache@v5
with:
path: ~/.cache/pypoetry
key: poetry-${{ runner.os }}-py3.12-libs-${{ hashFiles('autogpt_platform/autogpt_libs/poetry.lock') }}
- name: Install Poetry
run: |
HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry ../backend/poetry.lock)
echo "Using Poetry version ${HEAD_POETRY_VERSION}"
curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
- name: Install Python dependencies
run: poetry install
- name: Run pytest
run: poetry run pytest -q
env:
CI: true
PLAIN_OUTPUT: True
type-check:
permissions:
contents: read
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Set up Python dependency cache
uses: actions/cache@v5
with:
path: ~/.cache/pypoetry
key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
- name: Install Poetry
run: |
HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
echo "Using Poetry version ${HEAD_POETRY_VERSION}"
curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -
- name: Install Python dependencies
run: poetry install
- name: Generate Prisma Client
run: poetry run prisma generate && poetry run gen-prisma-stub
- name: Run Pyright
run: poetry run pyright --pythonversion ${{ matrix.python-version }}
env:
CI: true
PLAIN_OUTPUT: True
test:
permissions:
contents: read
# The suite is ~10.6k tests run serially (no pytest-xdist): the pytest step
# alone measures p50 ~13min / max ~18min, on top of 2-7min of container,
# checkout and dependency setup. A 20min cap sat *below* the job's real p95,
# so healthy runs were killed mid-suite and reported as `cancelled`. On
# merge_group that is fatal: `Check PR Status` treats a non-success check as
# a failure and GitHub ejects the PR from the merge queue. This is a guard
# against a hung job, not a performance budget - keep it well above p99.
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
runs-on: ubuntu-latest
services:
# Redis is provisioned as a real 3-shard cluster below via docker
# run (see the "Start Redis Cluster" step). GHA services can't
# override the image CMD or stand up multi-container clusters, so
# that setup is inlined — it mirrors the topology of the local dev
# compose stack (autogpt_platform/docker-compose.platform.yml) and
# prod helm chart.
rabbitmq:
image: rabbitmq:4.1.4
ports:
- 5672:5672
env:
RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
options: >-
--health-cmd "rabbitmq-diagnostics -q ping"
--health-interval 30s
--health-timeout 10s
--health-retries 5
--health-start-period 10s
clamav:
image: clamav/clamav-debian:latest
ports:
- 3310:3310
env:
CLAMAV_NO_FRESHCLAMD: false
CLAMD_CONF_StreamMaxLength: 50M
CLAMD_CONF_MaxFileSize: 100M
CLAMD_CONF_MaxScanSize: 100M
CLAMD_CONF_MaxThreads: 4
CLAMD_CONF_ReadTimeout: 300
options: >-
--health-cmd "clamdscan --version || exit 1"
--health-interval 30s
--health-timeout 10s
--health-retries 5
--health-start-period 180s
# FalkorDB backs the Graphiti knowledge graph the copilot uses for
# long-term memory. Mirrors the local compose mapping
# (autogpt_platform/docker-compose.platform.yml: ``6380:6379``) so
# ``GraphitiConfig`` defaults (``localhost:6380``,
# password=``local-dev-password`` from backend/.env.default) work
# unchanged. ``REDIS_ARGS=--requirepass <pw>`` is the FalkorDB
# image's documented way to set the bearer; mirrors the entrypoint
# the compose stack writes. Integration tests in
# ``backend/copilot/graphiti/*_integration_test.py`` skip cleanly
# via the ``falkordb_available`` fixture when the port isn't open
# (e.g. on a contributor's laptop) — this service makes the port
# available in CI so the integration suite actually runs.
# ``redis-cli`` ships inside the FalkorDB image; the healthcheck
# passes ``-a`` so AUTH succeeds rather than the daemon rejecting
# an unauthenticated PING.
# Pinned to a concrete release (unlike the local compose ``:latest``)
# so an upstream FalkorDB push can't start failing CI on unrelated
# PRs; bump deliberately alongside the compose stack.
falkordb:
image: falkordb/falkordb:v4.18.9
ports:
- 6380:6379
env:
REDIS_ARGS: "--requirepass local-dev-password"
options: >-
--health-cmd "redis-cli -a local-dev-password ping || exit 1"
--health-interval 10s
--health-timeout 5s
--health-retries 10
--health-start-period 10s
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
# Full ref history is needed so the "Install Poetry" step below can
# read poetry.lock off the base branch. `filter: blob:none` keeps
# every ref reachable while skipping the blob download for all of
# them, then lazily fetches the one blob that step actually reads.
# Without it this checkout has been measured at 429s vs 27s for a
# sibling matrix leg in the same run - enough variance on its own to
# blow the job budget.
fetch-depth: 0
filter: blob:none
submodules: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Start Redis Cluster (3 shards)
run: |
# 3-master Redis Cluster matching the local compose stack
# (autogpt_platform/docker-compose.platform.yml) and prod. Each
# shard runs in its own container on a dedicated bridge network,
# announces its compose-style hostname for intra-network clients,
# and publishes 1700N on the GHA host so tests can reach every
# shard via localhost. The backend's ``_address_remap`` rewrites
# every CLUSTER SLOTS reply to localhost:<announced-port>, which
# picks the right published port per shard.
#
# Not reusing docker-compose.platform.yml directly because compose
# validates the full file even when only some services are ``up``
# — pulling it in would needlessly couple CI to the full
# local-dev stack.
docker network create redis-cluster-ci
for i in 0 1 2; do
port=$((17000 + i))
bus=$((27000 + i))
docker run -d --name redis-$i --network redis-cluster-ci \
--network-alias redis-$i \
-p $port:$port \
redis:7 \
redis-server --port $port \
--cluster-enabled yes \
--cluster-config-file nodes.conf \
--cluster-node-timeout 5000 \
--cluster-require-full-coverage no \
--cluster-announce-hostname redis-$i \
--cluster-announce-port $port \
--cluster-announce-bus-port $bus \
--cluster-preferred-endpoint-type hostname
done
# Wait for each shard to accept commands.
for i in 0 1 2; do
port=$((17000 + i))
for _ in $(seq 1 30); do
docker exec redis-$i redis-cli -p $port ping 2>/dev/null | grep -q PONG && break
sleep 1
done
done
# Form the cluster from an init container on the same network so
# --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.
docker run --rm --network redis-cluster-ci redis:7 \
redis-cli --cluster create \
redis-0:17000 redis-1:17001 redis-2:17002 \
--cluster-replicas 0 --cluster-yes
# Confirm convergence.
for _ in $(seq 1 30); do
state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')
if [ "$state" = "ok" ]; then
echo "Redis Cluster ready (3 shards, state=ok)"
docker exec redis-0 redis-cli -p 17000 cluster nodes
exit 0
fi
sleep 1
done
echo "Redis Cluster failed to reach ok state" >&2
docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true
exit 1
- id: get_date
name: Get date
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Set up Python dependency cache
uses: actions/cache@v5
with:
path: ~/.cache/pypoetry
key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
- name: Install Poetry
run: |
# Extract Poetry version from backend/poetry.lock
HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
echo "Found Poetry version ${HEAD_POETRY_VERSION} in backend/poetry.lock"
if [ -n "$BASE_REF" ]; then
BASE_BRANCH=${BASE_REF/refs\/heads\//}
BASE_POETRY_VERSION=$((git show "origin/$BASE_BRANCH":./poetry.lock; true) | python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
echo "Found Poetry version ${BASE_POETRY_VERSION} in backend/poetry.lock on ${BASE_REF}"
POETRY_VERSION=$(printf '%s\n' "$HEAD_POETRY_VERSION" "$BASE_POETRY_VERSION" | sort -V | tail -n1)
else
POETRY_VERSION=$HEAD_POETRY_VERSION
fi
echo "Using Poetry version ${POETRY_VERSION}"
# Install Poetry
curl -sSL https://install.python-poetry.org | POETRY_VERSION=$POETRY_VERSION python3 -
if [ "${{ runner.os }}" = "macOS" ]; then
PATH="$HOME/.local/bin:$PATH"
echo "$HOME/.local/bin" >> $GITHUB_PATH
fi
env:
BASE_REF: ${{ github.base_ref || github.event.merge_group.base_ref }}
- name: Check poetry.lock
run: |
poetry lock
if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
echo "Error: poetry.lock not up to date."
echo
git diff poetry.lock
exit 1
fi
- name: Install Python dependencies
run: poetry install
- name: Generate Prisma Client
run: poetry run prisma generate && poetry run gen-prisma-stub
- name: Start Postgres
working-directory: autogpt_platform
run: |
# Plain pgvector Postgres; the init script creates the legacy
# `auth` schema shim that historical migrations reference.
docker run -d --name postgres-test \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=postgres \
-p 5432:5432 \
-v "$(pwd)/db/init/00-init.sql:/docker-entrypoint-initdb.d/00-init.sql:ro" \
pgvector/pgvector:pg15
for _ in $(seq 1 30); do
if docker exec postgres-test pg_isready -U postgres >/dev/null 2>&1; then
echo "Postgres ready"
exit 0
fi
sleep 1
done
echo "Postgres failed to start" >&2
docker logs postgres-test >&2
exit 1
- name: Wait for ClamAV to be ready
run: |
echo "Waiting for ClamAV daemon to start..."
max_attempts=60
attempt=0
until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; do
echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
sleep 5
attempt=$((attempt+1))
done
if [ $attempt -eq $max_attempts ]; then
echo "ClamAV failed to start after $((max_attempts*5)) seconds"
echo "Checking ClamAV service logs..."
docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
exit 1
fi
echo "ClamAV is ready!"
# Verify ClamAV is responsive
echo "Testing ClamAV connection..."
timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {
echo "ClamAV is not responding to PING"
docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
exit 1
}
- name: Run Database Migrations
run: poetry run prisma migrate deploy
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
- name: Run pytest with coverage
run: |
if [[ "${{ runner.debug }}" == "1" ]]; then
poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \
--cov=backend --cov-branch --cov-report term-missing --cov-report xml
else
poetry run pytest -s -vv \
--cov=backend --cov-branch --cov-report term-missing --cov-report xml
fi
env:
LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
JWT_VERIFY_KEY: "ci-only-jwt-verify-key-with-at-least-32-characters" # DO NOT USE IN PRODUCTION!!
REDIS_HOST: "localhost"
REDIS_PORT: "17000"
ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
# Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its
# own isolated 3-shard cluster (ports 27110–27112) and exercise
# ``docker restart <shard>`` mid-stream. Off locally so a
# contributor's ``poetry run test`` doesn't pay the ~15s cost.
E2E_RESTART_ISOLATED: "1"
- name: Upload coverage reports to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: platform-backend
files: ./autogpt_platform/backend/coverage.xml
# On a manual workflow_dispatch, attach the upload to the given PR so
# codecov re-evaluates that PR's project status against its base.
# Empty for all automatic events (push/pull_request/merge_group), so
# their normal PR/commit detection is unchanged.
override_pr: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || '' }}
env:
CI: true
PLAIN_OUTPUT: True
RUN_ENV: local
PORT: 8080
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
# We know these are here, don't report this as a security vulnerability
# This is used as the default credential for the entire system's RabbitMQ instance
# If you want to replace this, you can do so by making our entire system generate
# new credentials for each local user and update the environment variables in
# the backend service, docker composes, and examples
RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"