platform-backend-ci.yml

name: AutoGPT Platform - Backend CI

on:
  push:
    branches: [master, dev, ci-test*]
    paths:
      - ".github/workflows/platform-backend-ci.yml"
      - ".github/workflows/scripts/get_package_version_from_lockfile.py"
      - "autogpt_platform/backend/**"
      - "autogpt_platform/autogpt_libs/**"
  pull_request:
    branches: [master, dev, release-*]
    paths:
      - ".github/workflows/platform-backend-ci.yml"
      - ".github/workflows/scripts/get_package_version_from_lockfile.py"
      - "autogpt_platform/backend/**"
      - "autogpt_platform/autogpt_libs/**"
  merge_group:

concurrency:
  group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
  cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}

defaults:
  run:
    shell: bash
    working-directory: autogpt_platform/backend

jobs:
  lint:
    permissions:
      contents: read
    timeout-minutes: 10
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Set up Python dependency cache
        uses: actions/cache@v5
        with:
          path: ~/.cache/pypoetry
          key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

      - name: Install Poetry
        run: |
          HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
          echo "Using Poetry version ${HEAD_POETRY_VERSION}"
          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -

      - name: Install Python dependencies
        run: poetry install

      - name: Run Linters
        run: poetry run lint --skip-pyright

    env:
      CI: true
      PLAIN_OUTPUT: True

  type-check:
    permissions:
      contents: read
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up Python dependency cache
        uses: actions/cache@v5
        with:
          path: ~/.cache/pypoetry
          key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

      - name: Install Poetry
        run: |
          HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
          echo "Using Poetry version ${HEAD_POETRY_VERSION}"
          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$HEAD_POETRY_VERSION python3 -

      - name: Install Python dependencies
        run: poetry install

      - name: Generate Prisma Client
        run: poetry run prisma generate && poetry run gen-prisma-stub

      - name: Run Pyright
        run: poetry run pyright --pythonversion ${{ matrix.python-version }}

    env:
      CI: true
      PLAIN_OUTPUT: True

  test:
    permissions:
      contents: read
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    runs-on: ubuntu-latest

    services:
      # Redis is provisioned as a real 3-shard cluster below via docker
      # run (see the "Start Redis Cluster" step). GHA services can't
      # override the image CMD or stand up multi-container clusters, so
      # that setup is inlined — it mirrors the topology of the local dev
      # compose stack (autogpt_platform/docker-compose.platform.yml) and
      # prod helm chart.
      rabbitmq:
        image: rabbitmq:4.1.4
        ports:
          - 5672:5672
        env:
          RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
          RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
        options: >-
          --health-cmd "rabbitmq-diagnostics -q ping"
          --health-interval 30s
          --health-timeout 10s
          --health-retries 5
          --health-start-period 10s
      clamav:
        image: clamav/clamav-debian:latest
        ports:
          - 3310:3310
        env:
          CLAMAV_NO_FRESHCLAMD: false
          CLAMD_CONF_StreamMaxLength: 50M
          CLAMD_CONF_MaxFileSize: 100M
          CLAMD_CONF_MaxScanSize: 100M
          CLAMD_CONF_MaxThreads: 4
          CLAMD_CONF_ReadTimeout: 300
        options: >-
          --health-cmd "clamdscan --version || exit 1"
          --health-interval 30s
          --health-timeout 10s
          --health-retries 5
          --health-start-period 180s
      # FalkorDB backs the Graphiti knowledge graph the copilot uses for
      # long-term memory. Mirrors the local compose mapping
      # (autogpt_platform/docker-compose.platform.yml: ``6380:6379``) so
      # ``GraphitiConfig`` defaults (``localhost:6380``,
      # password=``local-dev-password`` from backend/.env.default) work
      # unchanged. ``REDIS_ARGS=--requirepass <pw>`` is the FalkorDB
      # image's documented way to set the bearer; mirrors the entrypoint
      # the compose stack writes. Integration tests in
      # ``backend/copilot/graphiti/*_integration_test.py`` skip cleanly
      # via the ``falkordb_available`` fixture when the port isn't open
      # (e.g. on a contributor's laptop) — this service makes the port
      # available in CI so the integration suite actually runs.
      # ``redis-cli`` ships inside the FalkorDB image; the healthcheck
      # passes ``-a`` so AUTH succeeds rather than the daemon rejecting
      # an unauthenticated PING.
      # Pinned to a concrete release (unlike the local compose ``:latest``)
      # so an upstream FalkorDB push can't start failing CI on unrelated
      # PRs; bump deliberately alongside the compose stack.
      falkordb:
        image: falkordb/falkordb:v4.18.9
        ports:
          - 6380:6379
        env:
          REDIS_ARGS: "--requirepass local-dev-password"
        options: >-
          --health-cmd "redis-cli -a local-dev-password ping || exit 1"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
          --health-start-period 10s

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          submodules: true

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Start Redis Cluster (3 shards)
        run: |
          # 3-master Redis Cluster matching the local compose stack
          # (autogpt_platform/docker-compose.platform.yml) and prod. Each
          # shard runs in its own container on a dedicated bridge network,
          # announces its compose-style hostname for intra-network clients,
          # and publishes 1700N on the GHA host so tests can reach every
          # shard via localhost. The backend's ``_address_remap`` rewrites
          # every CLUSTER SLOTS reply to localhost:<announced-port>, which
          # picks the right published port per shard.
          #
          # Not reusing docker-compose.platform.yml directly because compose
          # validates the full file even when only some services are ``up``,
          # and that file references services (db/kong/...) defined in a
          # sibling compose file — pulling both in would needlessly couple
          # CI to the full local-dev stack.
          docker network create redis-cluster-ci
          for i in 0 1 2; do
            port=$((17000 + i))
            bus=$((27000 + i))
            docker run -d --name redis-$i --network redis-cluster-ci \
              --network-alias redis-$i \
              -p $port:$port \
              redis:7 \
              redis-server --port $port \
                --cluster-enabled yes \
                --cluster-config-file nodes.conf \
                --cluster-node-timeout 5000 \
                --cluster-require-full-coverage no \
                --cluster-announce-hostname redis-$i \
                --cluster-announce-port $port \
                --cluster-announce-bus-port $bus \
                --cluster-preferred-endpoint-type hostname
          done
          # Wait for each shard to accept commands.
          for i in 0 1 2; do
            port=$((17000 + i))
            for _ in $(seq 1 30); do
              docker exec redis-$i redis-cli -p $port ping 2>/dev/null | grep -q PONG && break
              sleep 1
            done
          done
          # Form the cluster from an init container on the same network so
          # --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.
          docker run --rm --network redis-cluster-ci redis:7 \
            redis-cli --cluster create \
              redis-0:17000 redis-1:17001 redis-2:17002 \
              --cluster-replicas 0 --cluster-yes
          # Confirm convergence.
          for _ in $(seq 1 30); do
            state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')
            if [ "$state" = "ok" ]; then
              echo "Redis Cluster ready (3 shards, state=ok)"
              docker exec redis-0 redis-cli -p 17000 cluster nodes
              exit 0
            fi
            sleep 1
          done
          echo "Redis Cluster failed to reach ok state" >&2
          docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true
          exit 1

      - name: Setup Supabase
        uses: supabase/setup-cli@v1
        with:
          version: 1.178.1

      - id: get_date
        name: Get date
        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT

      - name: Set up Python dependency cache
        uses: actions/cache@v5
        with:
          path: ~/.cache/pypoetry
          key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}

      - name: Install Poetry
        run: |
          # Extract Poetry version from backend/poetry.lock
          HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
          echo "Found Poetry version ${HEAD_POETRY_VERSION} in backend/poetry.lock"

          if [ -n "$BASE_REF" ]; then
            BASE_BRANCH=${BASE_REF/refs\/heads\//}
            BASE_POETRY_VERSION=$((git show "origin/$BASE_BRANCH":./poetry.lock; true) | python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
            echo "Found Poetry version ${BASE_POETRY_VERSION} in backend/poetry.lock on ${BASE_REF}"
            POETRY_VERSION=$(printf '%s\n' "$HEAD_POETRY_VERSION" "$BASE_POETRY_VERSION" | sort -V | tail -n1)
          else
            POETRY_VERSION=$HEAD_POETRY_VERSION
          fi
          echo "Using Poetry version ${POETRY_VERSION}"

          # Install Poetry
          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$POETRY_VERSION python3 -

          if [ "${{ runner.os }}" = "macOS" ]; then
            PATH="$HOME/.local/bin:$PATH"
            echo "$HOME/.local/bin" >> $GITHUB_PATH
          fi
        env:
          BASE_REF: ${{ github.base_ref || github.event.merge_group.base_ref }}

      - name: Check poetry.lock
        run: |
          poetry lock

          if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
            echo "Error: poetry.lock not up to date."
            echo
            git diff poetry.lock
            exit 1
          fi

      - name: Install Python dependencies
        run: poetry install

      - name: Generate Prisma Client
        run: poetry run prisma generate && poetry run gen-prisma-stub

      - id: supabase
        name: Start Supabase
        working-directory: .
        run: |
          supabase init
          supabase start --exclude postgres-meta,realtime,storage-api,imgproxy,inbucket,studio,edge-runtime,logflare,vector,supavisor
          supabase status -o env | sed 's/="/=/; s/"$//' >> $GITHUB_OUTPUT
        # outputs:
        # DB_URL, API_URL, GRAPHQL_URL, ANON_KEY, SERVICE_ROLE_KEY, JWT_SECRET

      - name: Wait for ClamAV to be ready
        run: |
          echo "Waiting for ClamAV daemon to start..."
          max_attempts=60
          attempt=0

          until nc -z localhost 3310 || [ $attempt -eq $max_attempts ]; do
            echo "ClamAV is unavailable - sleeping (attempt $((attempt+1))/$max_attempts)"
            sleep 5
            attempt=$((attempt+1))
          done

          if [ $attempt -eq $max_attempts ]; then
            echo "ClamAV failed to start after $((max_attempts*5)) seconds"
            echo "Checking ClamAV service logs..."
            docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
            exit 1
          fi

          echo "ClamAV is ready!"

          # Verify ClamAV is responsive
          echo "Testing ClamAV connection..."
          timeout 10 bash -c 'echo "PING" | nc localhost 3310' || {
            echo "ClamAV is not responding to PING"
            docker logs $(docker ps -q --filter "ancestor=clamav/clamav-debian:latest") 2>&1 | tail -50 || echo "No ClamAV container found"
            exit 1
          }

      - name: Run Database Migrations
        run: poetry run prisma migrate deploy
        env:
          DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
          DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}

      - name: Run pytest with coverage
        run: |
          if [[ "${{ runner.debug }}" == "1" ]]; then
            poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \
              --cov=backend --cov-branch --cov-report term-missing --cov-report xml
          else
            poetry run pytest -s -vv \
              --cov=backend --cov-branch --cov-report term-missing --cov-report xml
          fi
        env:
          LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
          DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
          DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}
          SUPABASE_URL: ${{ steps.supabase.outputs.API_URL }}
          SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}
          JWT_VERIFY_KEY: ${{ steps.supabase.outputs.JWT_SECRET }}
          REDIS_HOST: "localhost"
          REDIS_PORT: "17000"
          ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
          # Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its
          # own isolated 3-shard cluster (ports 27110–27112) and exercise
          # ``docker restart <shard>`` mid-stream. Off locally so a
          # contributor's ``poetry run test`` doesn't pay the ~15s cost.
          E2E_RESTART_ISOLATED: "1"

      - name: Upload coverage reports to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          flags: platform-backend
          files: ./autogpt_platform/backend/coverage.xml

    env:
      CI: true
      PLAIN_OUTPUT: True
      RUN_ENV: local
      PORT: 8080
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      # We know these are here, don't report this as a security vulnerability
      # This is used as the default credential for the entire system's RabbitMQ instance
      # If you want to replace this, you can do so by making our entire system generate
      # new credentials for each local user and update the environment variables in
      # the backend service, docker composes, and examples
      RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
      RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"