feat(backend/copilot): Streamline AutoPilot agent creation #21767

Workflow file for this run

.github/workflows/platform-backend-ci.yml at 783275d

	name: AutoGPT Platform - Backend CI

	on:
	push:
	branches: [master, dev, ci-test*]
	paths:
	- ".github/workflows/platform-backend-ci.yml"
	- ".github/workflows/scripts/get_package_version_from_lockfile.py"
	- "autogpt_platform/backend/**"
	- "autogpt_platform/autogpt_libs/**"
	pull_request:
	branches: [master, dev, release-*]
	paths:
	- ".github/workflows/platform-backend-ci.yml"
	- ".github/workflows/scripts/get_package_version_from_lockfile.py"
	- "autogpt_platform/backend/**"
	- "autogpt_platform/autogpt_libs/**"
	merge_group:

	concurrency:
	group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) \|\| github.sha) }}
	cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}

	defaults:
	run:
	shell: bash
	working-directory: autogpt_platform/backend

	jobs:
	lint:
	permissions:
	contents: read
	timeout-minutes: 10
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v6

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"

	- name: Set up Python dependency cache
	uses: actions/cache@v5
	with:
	path: ~/.cache/pypoetry
	key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

	- name: Install Poetry
	run: \|
	HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
	echo "Using Poetry version ${HEAD_POETRY_VERSION}"
	curl -sSL https://install.python-poetry.org \| POETRY_VERSION=$HEAD_POETRY_VERSION python3 -

	- name: Install Python dependencies
	run: poetry install

	- name: Run Linters
	run: poetry run lint --skip-pyright

	env:
	CI: true
	PLAIN_OUTPUT: True

	type-check:
	permissions:
	contents: read
	timeout-minutes: 10
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.12", "3.13"]
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v6

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Set up Python dependency cache
	uses: actions/cache@v5
	with:
	path: ~/.cache/pypoetry
	key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

	- name: Install Poetry
	run: \|
	HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
	echo "Using Poetry version ${HEAD_POETRY_VERSION}"
	curl -sSL https://install.python-poetry.org \| POETRY_VERSION=$HEAD_POETRY_VERSION python3 -

	- name: Install Python dependencies
	run: poetry install

	- name: Generate Prisma Client
	run: poetry run prisma generate && poetry run gen-prisma-stub

	- name: Run Pyright
	run: poetry run pyright --pythonversion ${{ matrix.python-version }}

	env:
	CI: true
	PLAIN_OUTPUT: True

	test:
	permissions:
	contents: read
	timeout-minutes: 20
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.12", "3.13"]
	runs-on: ubuntu-latest

	services:
	# Redis is provisioned as a real 3-shard cluster below via docker
	# run (see the "Start Redis Cluster" step). GHA services can't
	# override the image CMD or stand up multi-container clusters, so
	# that setup is inlined — it mirrors the topology of the local dev
	# compose stack (autogpt_platform/docker-compose.platform.yml) and
	# prod helm chart.
	rabbitmq:
	image: rabbitmq:4.1.4
	ports:
	- 5672:5672
	env:
	RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
	RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
	options: >-
	--health-cmd "rabbitmq-diagnostics -q ping"
	--health-interval 30s
	--health-timeout 10s
	--health-retries 5
	--health-start-period 10s
	clamav:
	image: clamav/clamav-debian:latest
	ports:
	- 3310:3310
	env:
	CLAMAV_NO_FRESHCLAMD: false
	CLAMD_CONF_StreamMaxLength: 50M
	CLAMD_CONF_MaxFileSize: 100M
	CLAMD_CONF_MaxScanSize: 100M
	CLAMD_CONF_MaxThreads: 4
	CLAMD_CONF_ReadTimeout: 300
	options: >-
	--health-cmd "clamdscan --version \|\| exit 1"
	--health-interval 30s
	--health-timeout 10s
	--health-retries 5
	--health-start-period 180s
	# FalkorDB backs the Graphiti knowledge graph the copilot uses for
	# long-term memory. Mirrors the local compose mapping
	# (autogpt_platform/docker-compose.platform.yml: ``6380:6379``) so
	# ``GraphitiConfig`` defaults (``localhost:6380``,
	# password=``local-dev-password`` from backend/.env.default) work
	# unchanged. ``REDIS_ARGS=--requirepass <pw>`` is the FalkorDB
	# image's documented way to set the bearer; mirrors the entrypoint
	# the compose stack writes. Integration tests in
	# ``backend/copilot/graphiti/*_integration_test.py`` skip cleanly
	# via the ``falkordb_available`` fixture when the port isn't open
	# (e.g. on a contributor's laptop) — this service makes the port
	# available in CI so the integration suite actually runs.
	# ``redis-cli`` ships inside the FalkorDB image; the healthcheck
	# passes ``-a`` so AUTH succeeds rather than the daemon rejecting
	# an unauthenticated PING.
	# Pinned to a concrete release (unlike the local compose ``:latest``)
	# so an upstream FalkorDB push can't start failing CI on unrelated
	# PRs; bump deliberately alongside the compose stack.
	falkordb:
	image: falkordb/falkordb:v4.18.9
	ports:
	- 6380:6379
	env:
	REDIS_ARGS: "--requirepass local-dev-password"
	options: >-
	--health-cmd "redis-cli -a local-dev-password ping \|\| exit 1"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 10
	--health-start-period 10s

	steps:
	- name: Checkout repository
	uses: actions/checkout@v6
	with:
	fetch-depth: 0
	submodules: true

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Start Redis Cluster (3 shards)
	run: \|
	# 3-master Redis Cluster matching the local compose stack
	# (autogpt_platform/docker-compose.platform.yml) and prod. Each
	# shard runs in its own container on a dedicated bridge network,
	# announces its compose-style hostname for intra-network clients,
	# and publishes 1700N on the GHA host so tests can reach every
	# shard via localhost. The backend's ``_address_remap`` rewrites
	# every CLUSTER SLOTS reply to localhost:<announced-port>, which
	# picks the right published port per shard.
	#
	# Not reusing docker-compose.platform.yml directly because compose
	# validates the full file even when only some services are ``up``,
	# and that file references services (db/kong/...) defined in a
	# sibling compose file — pulling both in would needlessly couple
	# CI to the full local-dev stack.
	docker network create redis-cluster-ci
	for i in 0 1 2; do
	port=$((17000 + i))
	bus=$((27000 + i))
	docker run -d --name redis-$i --network redis-cluster-ci \
	--network-alias redis-$i \
	-p $port:$port \
	redis:7 \
	redis-server --port $port \
	--cluster-enabled yes \
	--cluster-config-file nodes.conf \
	--cluster-node-timeout 5000 \
	--cluster-require-full-coverage no \
	--cluster-announce-hostname redis-$i \
	--cluster-announce-port $port \
	--cluster-announce-bus-port $bus \
	--cluster-preferred-endpoint-type hostname
	done
	# Wait for each shard to accept commands.
	for i in 0 1 2; do
	port=$((17000 + i))
	for _ in $(seq 1 30); do
	docker exec redis-$i redis-cli -p $port ping 2>/dev/null \| grep -q PONG && break
	sleep 1
	done
	done
	# Form the cluster from an init container on the same network so
	# --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.
	docker run --rm --network redis-cluster-ci redis:7 \
	redis-cli --cluster create \
	redis-0:17000 redis-1:17001 redis-2:17002 \
	--cluster-replicas 0 --cluster-yes
	# Confirm convergence.
	for _ in $(seq 1 30); do
	state=$(docker exec redis-0 redis-cli -p 17000 cluster info \| awk -F: '/^cluster_state:/ {print $2}' \| tr -d '[:cntrl:]')
	if [ "$state" = "ok" ]; then
	echo "Redis Cluster ready (3 shards, state=ok)"
	docker exec redis-0 redis-cli -p 17000 cluster nodes
	exit 0
	fi
	sleep 1
	done
	echo "Redis Cluster failed to reach ok state" >&2
	docker exec redis-0 redis-cli -p 17000 cluster info >&2 \|\| true
	exit 1

	- name: Setup Supabase
	uses: supabase/setup-cli@v1
	with:
	version: 1.178.1

	- id: get_date
	name: Get date
	run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT

	- name: Set up Python dependency cache
	uses: actions/cache@v5
	with:
	path: ~/.cache/pypoetry
	key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

	- name: Install Poetry
	run: \|
	# Extract Poetry version from backend/poetry.lock
	HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
	echo "Found Poetry version ${HEAD_POETRY_VERSION} in backend/poetry.lock"

	if [ -n "$BASE_REF" ]; then
	BASE_BRANCH=${BASE_REF/refs\/heads\//}
	BASE_POETRY_VERSION=$((git show "origin/$BASE_BRANCH":./poetry.lock; true) \| python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
	echo "Found Poetry version ${BASE_POETRY_VERSION} in backend/poetry.lock on ${BASE_REF}"
	POETRY_VERSION=$(printf '%s\n' "$HEAD_POETRY_VERSION" "$BASE_POETRY_VERSION" \| sort -V \| tail -n1)
	else
	POETRY_VERSION=$HEAD_POETRY_VERSION
	fi
	echo "Using Poetry version ${POETRY_VERSION}"

	# Install Poetry
	curl -sSL https://install.python-poetry.org \| POETRY_VERSION=$POETRY_VERSION python3 -

	if [ "${{ runner.os }}" = "macOS" ]; then
	PATH="$HOME/.local/bin:$PATH"
	echo "$HOME/.local/bin" >> $GITHUB_PATH
	fi
	env:
	BASE_REF: ${{ github.base_ref \|\| github.event.merge_group.base_ref }}

	- name: Check poetry.lock
	run: \|
	poetry lock

	if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
	echo "Error: poetry.lock not up to date."
	echo
	git diff poetry.lock
	exit 1
	fi

	- name: Install Python dependencies
	run: poetry install

	- name: Generate Prisma Client
	run: poetry run prisma generate && poetry run gen-prisma-stub

	- id: supabase
	name: Start Supabase
	working-directory: .
	run: \|
	supabase init
	supabase start --exclude postgres-meta,realtime,storage-api,imgproxy,inbucket,studio,edge-runtime,logflare,vector,supavisor
	supabase status -o env \| sed 's/="/=/; s/"$//' >> $GITHUB_OUTPUT
	# outputs:
	# DB_URL, API_URL, GRAPHQL_URL, ANON_KEY, SERVICE_ROLE_KEY, JWT_SECRET

	- name: Wait for ClamAV to be ready
	run: \|
	echo "Waiting for ClamAV daemon to start..."
	max_attempts=60
	attempt=0

	until nc -z localhost 3310 \|\| [ $attempt -eq $max_attempts ]; do
	echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
	sleep 5
	attempt=$((attempt+1))
	done

	if [ $attempt -eq $max_attempts ]; then
	echo "ClamAV failed to start after $((max_attempts*5)) seconds"
	echo "Checking ClamAV service logs..."
	docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 \| tail -50 \|\| echo "No ClamAV container found"
	exit 1
	fi

	echo "ClamAV is ready!"

	# Verify ClamAV is responsive
	echo "Testing ClamAV connection..."
	timeout 10 bash -c 'echo "PING" \| nc localhost 3310' \|\| {
	echo "ClamAV is not responding to PING"
	docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 \| tail -50 \|\| echo "No ClamAV container found"
	exit 1
	}

	- name: Run Database Migrations
	run: poetry run prisma migrate deploy
	env:
	DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
	DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}

	- name: Run pytest with coverage
	run: \|
	if [[ "${{ runner.debug }}" == "1" ]]; then
	poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \
	--cov=backend --cov-branch --cov-report term-missing --cov-report xml
	else
	poetry run pytest -s -vv \
	--cov=backend --cov-branch --cov-report term-missing --cov-report xml
	fi
	env:
	LOG_LEVEL: ${{ runner.debug && 'DEBUG' \|\| 'INFO' }}
	DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
	DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}
	SUPABASE_URL: ${{ steps.supabase.outputs.API_URL }}
	SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}
	JWT_VERIFY_KEY: ${{ steps.supabase.outputs.JWT_SECRET }}
	REDIS_HOST: "localhost"
	REDIS_PORT: "17000"
	ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
	# Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its
	# own isolated 3-shard cluster (ports 27110–27112) and exercise
	# ``docker restart <shard>`` mid-stream. Off locally so a
	# contributor's ``poetry run test`` doesn't pay the ~15s cost.
	E2E_RESTART_ISOLATED: "1"

	- name: Upload coverage reports to Codecov
	if: ${{ !cancelled() }}
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	flags: platform-backend
	files: ./autogpt_platform/backend/coverage.xml

	env:
	CI: true
	PLAIN_OUTPUT: True
	RUN_ENV: local
	PORT: 8080
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	# We know these are here, don't report this as a security vulnerability
	# This is used as the default credential for the entire system's RabbitMQ instance
	# If you want to replace this, you can do so by making our entire system generate
	# new credentials for each local user and update the environment variables in
	# the backend service, docker composes, and examples
	RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
	RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"

Navigation Menu

Uh oh!

feat(backend/copilot): Streamline AutoPilot agent creation #21767

Workflow file

feat(backend/copilot): Streamline AutoPilot agent creation #21767

Uh oh!

Workflow file for this run