Apply credibility hardening in retrieval and CI benchmark gates #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main, master] | |
| pull_request: | |
| branches: [main, master] | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install package | |
| run: pip install -e ".[dev,signing,pdf,api]" | |
| - name: CLI smoke test | |
| run: vsa --help | |
| - name: Build example report | |
| run: vsa build examples/brca1_input.json --out reports/brca1_report.json --claim-mode rule | |
| - name: Validate good examples | |
| run: vsa validate reports/brca1_report.json | |
| - name: Validate bad examples fail | |
| run: | | |
| vsa validate examples/bad_unsupported_claim.json --skip-hash-check && exit 1 || true | |
| vsa validate examples/bad_missing_evidence_ref.json --skip-hash-check && exit 1 || true | |
| - name: Audit report (rule mode) | |
| run: vsa audit reports/brca1_report.json --audit-mode rule --out reports/audit.json | |
| - name: SLSA attestation | |
| run: | | |
| vsa attest reports/brca1_report.json --out reports/attestation.json --subject-name report.json | |
| vsa verify-attestation reports/brca1_report.json reports/attestation.json --subject-name report.json | |
| - name: Export artifact bundle | |
| run: vsa export reports/brca1_report.json --out-dir reports/bundle --audit-mode rule | |
| - name: Verify export bundle | |
| run: vsa verify-bundle reports/bundle | |
| - name: Review workflow smoke | |
| run: | | |
| vsa review start reports/brca1_report.json --reviewer ci@example.com --out /tmp/review_started.json | |
| vsa review approve-claim /tmp/review_started.json --reviewer ci@example.com --claim C002 --out /tmp/reviewed.json | |
| vsa review verify /tmp/reviewed.json | |
| vsa validate /tmp/reviewed.json | |
| - name: API smoke test | |
| run: python -c "from fastapi.testclient import TestClient; from vsa.api.app import create_app; c=TestClient(create_app()); assert c.get('/health').json()['status']=='ok'" | |
| - name: Sign and verify report | |
| run: | | |
| vsa sign --generate-key --key-file /tmp/vsa_test_key | |
| vsa sign reports/brca1_report.json --key-file /tmp/vsa_test_key --out /tmp/signed_report.json | |
| vsa verify-signature /tmp/signed_report.json | |
| - name: Render test | |
| run: vsa render reports/brca1_report.json --format markdown --out reports/brca1_report.md | |
| - name: Hash reproducibility test | |
| run: | | |
| vsa hash reports/brca1_report.json --json > /tmp/hash1.json | |
| vsa hash reports/brca1_report.json --json > /tmp/hash2.json | |
| diff /tmp/hash1.json /tmp/hash2.json | |
| - name: Run pytest | |
| run: pytest --cov=vsa --cov-report=term-missing | |
| - name: Run benchmarks (offline) | |
| run: | | |
| vsa benchmark --out reports/benchmark_summary.json | |
| python -c "import json,sys; s=json.load(open('reports/benchmark_summary.json')); assert not s.get('regression'), s; assert not any(s.get('category_gaps',{}).values()), s['category_gaps']; assert s['total']>=50" | |
| - name: Upload CI artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vsa-reports-py${{ matrix.python-version }} | |
| path: | | |
| reports/brca1_report.json | |
| reports/brca1_report.md | |
| reports/audit.json | |
| reports/benchmark_summary.json | |
| reports/attestation.json | |
| reports/bundle/ | |
| macos-smoke: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install and smoke | |
| run: | | |
| pip install -e ".[dev,signing,api]" | |
| make test | |
| vsa benchmark --out reports/benchmark_summary.json | |
| acceptance: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Clean-clone acceptance bar | |
| run: bash scripts/acceptance.sh |