action.yml

name: 'EvalView - AI Agent Testing'
description: 'Catch agent regressions in CI. Detects tool changes, output drift, cost spikes. Works with LangGraph, CrewAI, Claude, GPT.'
author: 'EvalView Team'

branding:
  icon: 'check-circle'
  color: 'purple'

inputs:
  mode:
    description: 'Testing mode: "check" for regression detection (default), "run" for full evaluation'
    required: false
    default: 'check'
  command:
    description: 'Alias for "mode" — accepts "check" or "run". If both are set, "command" takes precedence.'
    required: false
  openai-api-key:
    description: 'OpenAI API key for LLM-as-judge evaluation'
    required: false
  anthropic-api-key:
    description: 'Anthropic API key (optional alternative to OpenAI)'
    required: false
  test-path:
    description: 'Path to test cases directory'
    required: false
    default: 'tests'
  config-path:
    description: 'Path to EvalView config file'
    required: false
    default: '.evalview/config.yaml'
  filter:
    description: 'Filter tests by name pattern'
    required: false
  max-workers:
    description: 'Maximum parallel workers'
    required: false
    default: '4'
  max-retries:
    description: 'Maximum retries for failed tests'
    required: false
    default: '2'
  fail-on-error:
    description: 'Fail the workflow if any test fails or regressions detected'
    required: false
    default: 'true'
  fail-on:
    description: 'Comma-separated statuses that fail CI (REGRESSION, TOOLS_CHANGED, OUTPUT_CHANGED, CONTRACT_DRIFT)'
    required: false
    default: 'REGRESSION'
  strict:
    description: 'Fail on any change (REGRESSION, TOOLS_CHANGED, OUTPUT_CHANGED)'
    required: false
    default: 'false'
  diff:
    description: 'Compare against golden baselines in "run" mode (ignored in "check" mode which always diffs)'
    required: false
    default: 'false'
  contracts:
    description: 'Check MCP contracts for interface drift before running tests'
    required: false
    default: 'false'
  generate-report:
    description: 'Generate HTML report'
    required: false
    default: 'true'
  post-comment:
    description: 'Automatically post results as PR comment (requires GITHUB_TOKEN)'
    required: false
    default: 'true'
  upload-artifact:
    description: 'Upload results and report as workflow artifacts'
    required: false
    default: 'true'
  evalview-version:
    description: 'EvalView version to install (e.g. "0.5.3", default: latest)'
    required: false
    default: ''
  python-version:
    description: 'Python version to use'
    required: false
    default: '3.11'

outputs:
  results-file:
    description: 'Path to JSON results file'
    value: ${{ steps.run-tests.outputs.results-file }}
  report-file:
    description: 'Path to HTML report file'
    value: ${{ steps.run-tests.outputs.report-file }}
  total-tests:
    description: 'Total number of tests run'
    value: ${{ steps.run-tests.outputs.total-tests }}
  passed-tests:
    description: 'Number of passed/unchanged tests'
    value: ${{ steps.run-tests.outputs.passed-tests }}
  failed-tests:
    description: 'Number of failed/regressed tests'
    value: ${{ steps.run-tests.outputs.failed-tests }}
  regressions:
    description: 'Number of regressions detected (check mode)'
    value: ${{ steps.run-tests.outputs.regressions }}
  tools-changed:
    description: 'Number of tests with tool changes (check mode)'
    value: ${{ steps.run-tests.outputs.tools-changed }}
  pass-rate:
    description: 'Pass rate percentage'
    value: ${{ steps.run-tests.outputs.pass-rate }}
  status:
    description: 'Overall status: passed, regression, tools_changed, output_changed'
    value: ${{ steps.run-tests.outputs.status }}

runs:
  using: 'composite'
  steps:
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ inputs.python-version }}

    - name: Install EvalView
      shell: bash
      run: |
        python -m pip install --upgrade pip
        if [ -n "${{ inputs.evalview-version }}" ]; then
          pip install "evalview==${{ inputs.evalview-version }}"
        else
          pip install evalview
        fi

    - name: Run EvalView Tests
      id: run-tests
      shell: bash
      env:
        OPENAI_API_KEY: ${{ inputs.openai-api-key }}
        ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
        INPUT_MODE: ${{ inputs.command || inputs.mode }}
        INPUT_TEST_PATH: ${{ inputs.test-path }}
        INPUT_FAIL_ON: ${{ inputs.fail-on }}
        INPUT_STRICT: ${{ inputs.strict }}
        INPUT_FILTER: ${{ inputs.filter }}
        INPUT_MAX_WORKERS: ${{ inputs.max-workers }}
        INPUT_MAX_RETRIES: ${{ inputs.max-retries }}
        INPUT_DIFF: ${{ inputs.diff }}
        INPUT_CONTRACTS: ${{ inputs.contracts }}
        INPUT_GENERATE_REPORT: ${{ inputs.generate-report }}
        INPUT_FAIL_ON_ERROR: ${{ inputs.fail-on-error }}
      run: |
        REPORT_PATH=""
        if [ "$INPUT_GENERATE_REPORT" = "true" ]; then
          REPORT_PATH="evalview-report.html"
        fi

        if [ "$INPUT_MODE" = "check" ]; then
          # ── Check mode (default): regression detection against baselines ──
          ARGS=("$INPUT_TEST_PATH" "--json" "--fail-on" "$INPUT_FAIL_ON")

          if [ "$INPUT_STRICT" = "true" ]; then
            ARGS+=("--strict")
          fi

          if [ -n "$REPORT_PATH" ]; then
            ARGS+=("--report" "$REPORT_PATH")
          fi

          if [ -n "$INPUT_FILTER" ]; then
            ARGS+=("--test" "$INPUT_FILTER")
          fi

          # Run check — capture JSON on stdout, let stderr pass through
          RESULTS_FILE=".evalview/results/check-ci.json"
          mkdir -p .evalview/results

          set +e
          evalview check "${ARGS[@]}" > "$RESULTS_FILE"
          EXIT_CODE=$?
          set -e

          echo "results-file=$RESULTS_FILE" >> "$GITHUB_OUTPUT"

          # Parse check JSON output
          TOTAL=$(jq -r '.summary.total_tests // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
          UNCHANGED=$(jq -r '.summary.unchanged // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
          REGRESSIONS=$(jq -r '.summary.regressions // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
          TOOLS_CHANGED=$(jq -r '.summary.tools_changed // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
          OUTPUT_CHANGED=$(jq -r '.summary.output_changed // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
          FAILED=$((REGRESSIONS + TOOLS_CHANGED + OUTPUT_CHANGED))

          if [ "$TOTAL" -gt 0 ]; then
            PASS_RATE=$(echo "scale=1; $UNCHANGED * 100 / $TOTAL" | bc)
          else
            PASS_RATE="0"
          fi

          # Determine overall status
          if [ "$REGRESSIONS" -gt 0 ]; then
            STATUS="regression"
          elif [ "$TOOLS_CHANGED" -gt 0 ]; then
            STATUS="tools_changed"
          elif [ "$OUTPUT_CHANGED" -gt 0 ]; then
            STATUS="output_changed"
          else
            STATUS="passed"
          fi

          echo "total-tests=$TOTAL" >> "$GITHUB_OUTPUT"
          echo "passed-tests=$UNCHANGED" >> "$GITHUB_OUTPUT"
          echo "failed-tests=$FAILED" >> "$GITHUB_OUTPUT"
          echo "regressions=$REGRESSIONS" >> "$GITHUB_OUTPUT"
          echo "tools-changed=$TOOLS_CHANGED" >> "$GITHUB_OUTPUT"
          echo "pass-rate=$PASS_RATE" >> "$GITHUB_OUTPUT"
          echo "status=$STATUS" >> "$GITHUB_OUTPUT"

        else
          # ── Run mode: full evaluation with scoring ──
          ARGS=("--max-workers" "$INPUT_MAX_WORKERS" "--max-retries" "$INPUT_MAX_RETRIES")

          if [ -n "$INPUT_FILTER" ]; then
            ARGS+=("--filter" "$INPUT_FILTER")
          fi

          if [ "$INPUT_DIFF" = "true" ]; then
            ARGS+=("--diff" "--fail-on" "$INPUT_FAIL_ON")
          fi

          if [ "$INPUT_CONTRACTS" = "true" ]; then
            ARGS+=("--contracts")
          fi

          if [ -n "$REPORT_PATH" ]; then
            ARGS+=("--html-report" "$REPORT_PATH")
          fi

          set +e
          evalview run "${ARGS[@]}"
          EXIT_CODE=$?
          set -e

          # Find results file
          RESULTS_FILE=$(ls -t .evalview/results/*.json 2>/dev/null | head -1)

          if [ -n "$RESULTS_FILE" ]; then
            echo "results-file=$RESULTS_FILE" >> "$GITHUB_OUTPUT"

            TOTAL=$(jq '.summary.total // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
            PASSED=$(jq '.summary.passed // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")
            FAILED=$(jq '.summary.failed // 0' "$RESULTS_FILE" 2>/dev/null || echo "0")

            if [ "$TOTAL" -gt 0 ]; then
              PASS_RATE=$(echo "scale=1; $PASSED * 100 / $TOTAL" | bc)
            else
              PASS_RATE="0"
            fi

            # Derive status from results
            if [ "$FAILED" -gt 0 ]; then
              STATUS="regression"
            else
              STATUS="passed"
            fi

            echo "total-tests=$TOTAL" >> "$GITHUB_OUTPUT"
            echo "passed-tests=$PASSED" >> "$GITHUB_OUTPUT"
            echo "failed-tests=$FAILED" >> "$GITHUB_OUTPUT"
            echo "pass-rate=$PASS_RATE" >> "$GITHUB_OUTPUT"
            echo "status=$STATUS" >> "$GITHUB_OUTPUT"
          fi
        fi

        if [ -f "$REPORT_PATH" ]; then
          echo "report-file=$REPORT_PATH" >> "$GITHUB_OUTPUT"
        fi

        # Save exit code for later — comment/summary steps run first
        echo "exit-code=$EXIT_CODE" >> "$GITHUB_OUTPUT"

    - name: Post PR comment & job summary
      if: always() && inputs.post-comment == 'true'
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        # Post comment (updates existing or creates new; gracefully no-ops outside PRs)
        evalview ci comment 2>/dev/null || true

    - name: Upload artifacts
      if: always() && inputs.upload-artifact == 'true'
      uses: actions/upload-artifact@v4
      with:
        name: evalview-results
        path: |
          .evalview/results/*.json
          evalview-report.html
        retention-days: 30
        if-no-files-found: ignore

    - name: Exit with test result
      if: always()
      shell: bash
      run: |
        EXIT_CODE="${{ steps.run-tests.outputs.exit-code }}"
        if [ "${{ inputs.fail-on-error }}" = "true" ] && [ "${EXIT_CODE:-0}" -ne 0 ]; then
          exit ${EXIT_CODE}
        fi