benchmark.yml

name: Virtual MCP Benchmark

on:
  # Post comment on PR open
  pull_request_target:
    types: [opened, synchronize]
    paths:
      - "apps/mesh/**"
      - "packages/**"

  # Manual trigger with options
  workflow_dispatch:
    inputs:
      mode:
        description: 'Benchmark mode'
        required: true
        default: 'quick'
        type: choice
        options:
          - quick
          - high
          - full
      pr_number:
        description: 'PR number to comment results on (optional)'
        required: false
        type: string

permissions:
  contents: read
  pull-requests: write
  actions: read

jobs:
  # Job 1: Post comment asking about benchmark (on PR open)
  ask-benchmark:
    if: github.event_name == 'pull_request_target' && github.event.action == 'opened'
    runs-on: ubuntu-latest
    steps:
      - name: Post benchmark question
        uses: peter-evans/create-or-update-comment@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ## 🧪 Benchmark

            Should we run the Virtual MCP strategy benchmark for this PR?

            React with :+1: to run the benchmark.

            | Reaction | Action |
            |----------|--------|
            | :+1: | Run quick benchmark (10 & 128 tools) |

            *Benchmark will run on the next push after you react.*

  # Job 2: Check for reactions on each push
  check-benchmark-approval:
    if: github.event_name == 'pull_request_target' && github.event.action == 'synchronize'
    runs-on: ubuntu-latest
    outputs:
      should_run: ${{ steps.check_reactions.outputs.should_run }}
      comment_id: ${{ steps.check_reactions.outputs.comment_id }}
    steps:
      - name: Check for benchmark reactions
        id: check_reactions
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          # Fetch comments and find the Benchmark comment
          COMMENT_DATA=$(gh api graphql -f query='
            query {
              repository(owner:"${{ github.repository_owner }}", name:"${{ github.event.repository.name }}") {
                pullRequest(number: '${PR_NUMBER}') {
                  comments(last: 100) {
                    nodes {
                      id
                      databaseId
                      body
                      reactions(last: 100) {
                        nodes {
                          content
                          user {
                            login
                          }
                        }
                      }
                    }
                  }
                }
              }
            }')

          # Check if the benchmark comment exists and has :+1: reaction
          THUMBS_UP=$(echo "$COMMENT_DATA" | jq -r '
            .data.repository.pullRequest.comments.nodes[] |
            select(.body | contains("## 🧪 Benchmark")) |
            .reactions.nodes[] |
            select(.content == "THUMBS_UP") |
            .user.login' | head -1)

          COMMENT_ID=$(echo "$COMMENT_DATA" | jq -r '
            .data.repository.pullRequest.comments.nodes[] |
            select(.body | contains("## 🧪 Benchmark")) |
            .databaseId')

          echo "comment_id=$COMMENT_ID" >> $GITHUB_OUTPUT

          if [ -n "$THUMBS_UP" ] && [ "$THUMBS_UP" != "null" ]; then
            echo "Found :+1: reaction from $THUMBS_UP"
            echo "should_run=true" >> $GITHUB_OUTPUT
          else
            echo "No :+1: reaction found on benchmark comment"
            echo "should_run=false" >> $GITHUB_OUTPUT
          fi

  # Job 3: Run benchmark (if approved via reaction or manual trigger)
  # SECURITY: For pull_request_target, we ONLY checkout the trusted base branch code
  # to prevent Pwn Request attacks where malicious PR code could steal secrets.
  # The benchmark runs against the base branch's benchmark implementation.
  benchmark:
    name: Run Benchmark
    needs: [check-benchmark-approval]
    if: |
      always() && (
        (needs.check-benchmark-approval.result == 'success' && needs.check-benchmark-approval.outputs.should_run == 'true') ||
        github.event_name == 'workflow_dispatch'
      )
    runs-on: ubuntu-latest
    timeout-minutes: 60

    steps:
      # SECURITY: Always checkout the base branch (trusted code), NOT the PR head
      # This prevents attackers from modifying package.json or scripts to steal secrets
      - name: Checkout repository (trusted base branch)
        uses: actions/checkout@v4
        with:
          # For PRs: checkout base branch (main). For workflow_dispatch: checkout default branch
          ref: ${{ github.event.pull_request.base.sha || github.sha }}

      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - name: Install dependencies
        run: bun install

      - name: Run benchmark
        env:
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
        run: |
          MODE="${{ github.event.inputs.mode || 'quick' }}"

          case $MODE in
            quick)
              bun run --cwd apps/benchmark benchmark:quick
              ;;
            high)
              bun run --cwd apps/benchmark benchmark:high
              ;;
            full)
              bun run --cwd apps/benchmark benchmark
              ;;
          esac

      - name: Find latest result
        id: find-result
        run: |
          RESULT_FILE=$(ls -t apps/benchmark/results/*.md | head -1)
          echo "result_file=$RESULT_FILE" >> $GITHUB_OUTPUT
          echo "result_name=$(basename $RESULT_FILE)" >> $GITHUB_OUTPUT

      - name: Output results to summary
        run: |
          echo "# 📊 Virtual MCP Benchmark Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          cat "${{ steps.find-result.outputs.result_file }}" >> $GITHUB_STEP_SUMMARY

      - name: Upload benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-${{ github.run_number }}
          path: apps/benchmark/results/*.md
          retention-days: 90

      - name: Comment results on PR
        if: github.event_name == 'pull_request_target' || github.event.inputs.pr_number != ''
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const resultFile = '${{ steps.find-result.outputs.result_file }}';
            const content = fs.readFileSync(resultFile, 'utf8');

            // Truncate if too long
            const maxLength = 60000;
            const truncated = content.length > maxLength
              ? content.substring(0, maxLength) + '\n\n... (truncated)'
              : content;

            const prNumber = context.payload.pull_request?.number || '${{ github.event.inputs.pr_number }}';

            if (prNumber) {
              await github.rest.issues.createComment({
                issue_number: parseInt(prNumber),
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: `## 📊 Benchmark Results\n\n${truncated}`
              });
            }