Updating results #206
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: validate-translations | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v5 | |
| with: | |
| token: ${{ secrets.SUBMODULE_PAT }} | |
| - name: Check out submodules | |
| run: | | |
| git config --global url."https://x-access-token:${{ secrets.SUBMODULE_PAT }}@github.com/".insteadOf "https://github.com/" | |
| git submodule update --init --depth=1 results test-corpus crust-bench | |
| - name: Unzip CRUST-bench datasets | |
| run: cd crust-bench/datasets && unzip -q CRUST_bench.zip | |
| - name: Set up Rust | |
| run: | | |
| rustup toolchain install 1.94.0 --profile default | |
| rustup toolchain install nightly --profile default | |
| rustup default 1.94.0 | |
| - name: Build harvest-tools | |
| run: cargo build --release --manifest-path tools/Cargo.toml | |
| - name: Cache workspace | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| . | |
| !.git | |
| key: workspace-${{ github.sha }} | |
| validate: | |
| needs: setup | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: CRUST kiro | |
| guard: results/CRUST/kiro | |
| cmd: ./tools/target/release/harvest-tools --agent kiro test CRUST --check | |
| needs_openssl: true | |
| needs_crust: true | |
| - name: CRUST-blind kiro | |
| guard: results/CRUST-blind/kiro | |
| cmd: ./tools/target/release/harvest-tools --agent kiro test CRUST --blind --check | |
| needs_openssl: true | |
| needs_crust: true | |
| - name: Test-Corpus kiro | |
| guard: results/Test-Corpus/kiro | |
| cmd: ./tools/target/release/harvest-tools --agent kiro test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus c2rust | |
| guard: results/Test-Corpus/c2rust | |
| cmd: ./tools/target/release/harvest-tools --agent c2rust test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus claude | |
| guard: results/Test-Corpus/claude | |
| cmd: ./tools/target/release/harvest-tools --agent claude test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus kiro-translate | |
| guard: results/Test-Corpus/kiro-translate | |
| cmd: ./tools/target/release/harvest-tools --agent kiro-translate test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: CRUST-blind kiro-translate | |
| guard: results/CRUST-blind/kiro-translate | |
| cmd: ./tools/target/release/harvest-tools --agent kiro-translate test CRUST --blind --check | |
| needs_openssl: true | |
| needs_crust: true | |
| - name: Test-Corpus laertes | |
| guard: results/Test-Corpus/laertes | |
| cmd: ./tools/target/release/harvest-tools --agent laertes test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus kimi | |
| guard: results/Test-Corpus/kimi | |
| cmd: ./tools/target/release/harvest-tools --agent kimi test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus gpt-5.4 | |
| guard: results/Test-Corpus/gpt-5.4 | |
| cmd: ./tools/target/release/harvest-tools --agent oneshot --model openai/gpt-5.4 test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| - name: Test-Corpus gemini-3.1-pro-preview | |
| guard: results/Test-Corpus/gemini-3.1-pro-preview | |
| cmd: ./tools/target/release/harvest-tools --agent oneshot --model google/gemini-3.1-pro-preview test all --check | |
| needs_openssl: false | |
| needs_crust: false | |
| name: ${{ matrix.name }} | |
| steps: | |
| - name: Restore workspace | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| . | |
| !.git | |
| key: workspace-${{ github.sha }} | |
| - name: Set up Rust | |
| run: | | |
| rustup toolchain install 1.94.0 --profile default | |
| rustup toolchain install nightly --profile default | |
| rustup default 1.94.0 | |
| - name: Install system dependencies | |
| if: matrix.needs_openssl | |
| run: sudo apt-get update && sudo apt-get install -y libssl-dev cmake pkg-config | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Validate | |
| env: | |
| CARGO_BUILD_JOBS: "2" | |
| run: | | |
| echo "::group::Environment diagnostics" | |
| echo "Guard path: ${{ matrix.guard }}" | |
| ls -la "${{ matrix.guard }}" 2>/dev/null | head -5 || echo "Guard path does not exist" | |
| echo "pkg-config openssl:" | |
| pkg-config --modversion openssl 2>&1 || echo "pkg-config not available" | |
| echo "openssl version:" | |
| openssl version 2>&1 || echo "openssl not found" | |
| echo "VaultSync .vsync check:" | |
| ls -la results/CRUST/kiro/VaultSync/.vsync 2>/dev/null && echo "WARNING: .vsync exists!" || echo ".vsync not present (good)" | |
| echo "c_blind_rsa result.json:" | |
| cat results/CRUST/kiro/c_blind_rsa_signatures/result.json 2>/dev/null || echo "not found" | |
| echo "VaultSync result.json:" | |
| cat results/CRUST/kiro/VaultSync/result.json 2>/dev/null || echo "not found" | |
| echo "::endgroup::" | |
| if [ -d "${{ matrix.guard }}" ]; then | |
| for attempt in 1 2 3; do | |
| echo "::group::Attempt $attempt/3" | |
| if ${{ matrix.cmd }}; then | |
| echo "::endgroup::" | |
| exit 0 | |
| fi | |
| echo "::endgroup::" | |
| if [ "$attempt" -lt 3 ]; then | |
| echo "⚠️ Attempt $attempt failed, retrying (flaky floating-point tests)..." | |
| find results -name target -type d -exec rm -rf {} + 2>/dev/null || true | |
| fi | |
| done | |
| echo "❌ Failed after 3 attempts" | |
| exit 1 | |
| fi |