fix: resolve installer failures and add comprehensive validation #43
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate Test Suites | |
| on: | |
| push: | |
| paths: | |
| - 'evals/agents/*/config/**/*.json' | |
| - 'evals/agents/*/tests/**/*.yaml' | |
| - 'scripts/validation/validate-test-suites.sh' | |
| - '.github/workflows/validate-test-suites.yml' | |
| pull_request: | |
| paths: | |
| - 'evals/agents/*/config/**/*.json' | |
| - 'evals/agents/*/tests/**/*.yaml' | |
| - 'scripts/validation/validate-test-suites.sh' | |
| - '.github/workflows/validate-test-suites.yml' | |
| workflow_dispatch: | |
| jobs: | |
| validate: | |
| name: Validate Test Suite Definitions | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: 'package-lock.json' | |
| - name: Install dependencies | |
| run: | | |
| npm ci | |
| - name: Validate all test suites | |
| run: | | |
| cd evals/framework | |
| npm run validate:suites:all | |
| - name: Comment on PR (if validation failed) | |
| if: failure() && github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: '❌ **Test Suite Validation Failed**\n\nPlease check the test suite JSON files for errors. Run `npm run validate:suites` locally to see details.' | |
| }) | |
| - name: Upload validation report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: suite-validation-report | |
| path: | | |
| evals/agents/*/config/**/*.json | |
| scripts/validation/validate-test-suites.sh | |
| retention-days: 7 |