Testing: E2E (Chained) #17787
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Testing: E2E (Chained)' | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| merge_group: | |
| workflow_run: | |
| workflows: ['Trigger E2E'] | |
| types: ['completed'] | |
| workflow_dispatch: | |
| inputs: | |
| head_sha: | |
| description: 'SHA of the commit to test' | |
| required: true | |
| repo_name: | |
| description: 'Repository name (e.g., owner/repo)' | |
| required: true | |
| concurrency: | |
| group: '${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.ref }}' | |
| cancel-in-progress: |- | |
| ${{ github.event_name != 'push' && github.event_name != 'merge_group' }} | |
| permissions: | |
| contents: 'read' | |
| statuses: 'write' | |
| jobs: | |
| merge_queue_skipper: | |
| name: 'Merge Queue Skipper' | |
| permissions: 'read-all' | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| if: "github.repository == 'google-gemini/gemini-cli'" | |
| outputs: | |
| skip: '${{ steps.merge-queue-e2e-skipper.outputs.skip-check }}' | |
| steps: | |
| - id: 'merge-queue-e2e-skipper' | |
| uses: 'cariad-tech/merge-queue-ci-skipper@1032489e59437862c90a08a2c92809c903883772' # ratchet:cariad-tech/merge-queue-ci-skipper@main | |
| with: | |
| secret: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}' | |
| continue-on-error: true | |
| download_repo_name: | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| if: "github.repository == 'google-gemini/gemini-cli' && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run')" | |
| outputs: | |
| repo_name: '${{ steps.output-repo-name.outputs.repo_name }}' | |
| head_sha: '${{ steps.output-repo-name.outputs.head_sha }}' | |
| steps: | |
| - name: 'Mock Repo Artifact' | |
| if: "${{ github.event_name == 'workflow_dispatch' }}" | |
| env: | |
| REPO_NAME: '${{ github.event.inputs.repo_name }}' | |
| run: | | |
| mkdir -p ./pr | |
| echo "${REPO_NAME}" > ./pr/repo_name | |
| - uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4 | |
| with: | |
| name: 'repo_name' | |
| path: 'pr/' | |
| - name: 'Download the repo_name artifact' | |
| uses: 'actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0' # ratchet:actions/download-artifact@v5 | |
| env: | |
| RUN_ID: "${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.run_id }}" | |
| with: | |
| github-token: '${{ secrets.GITHUB_TOKEN }}' | |
| name: 'repo_name' | |
| run-id: '${{ env.RUN_ID }}' | |
| path: '${{ runner.temp }}/artifacts' | |
| - name: 'Output Repo Name and SHA' | |
| id: 'output-repo-name' | |
| uses: 'actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd' # ratchet:actions/github-script@v8 | |
| with: | |
| github-token: '${{ secrets.GITHUB_TOKEN }}' | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const temp = '${{ runner.temp }}/artifacts'; | |
| const repoPath = path.join(temp, 'repo_name'); | |
| if (fs.existsSync(repoPath)) { | |
| const repo_name = String(fs.readFileSync(repoPath)).trim(); | |
| core.setOutput('repo_name', repo_name); | |
| } | |
| const shaPath = path.join(temp, 'head_sha'); | |
| if (fs.existsSync(shaPath)) { | |
| const head_sha = String(fs.readFileSync(shaPath)).trim(); | |
| core.setOutput('head_sha', head_sha); | |
| } | |
| parse_run_context: | |
| name: 'Parse run context' | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| needs: 'download_repo_name' | |
| if: "github.repository == 'google-gemini/gemini-cli' && always()" | |
| outputs: | |
| repository: '${{ steps.set_context.outputs.REPO }}' | |
| sha: '${{ steps.set_context.outputs.SHA }}' | |
| steps: | |
| - id: 'set_context' | |
| name: 'Set dynamic repository and SHA' | |
| env: | |
| REPO: '${{ needs.download_repo_name.outputs.repo_name || github.repository }}' | |
| SHA: '${{ needs.download_repo_name.outputs.head_sha || github.event.inputs.head_sha || github.event.workflow_run.head_sha || github.sha }}' | |
| shell: 'bash' | |
| run: | | |
| echo "REPO=$REPO" >> "$GITHUB_OUTPUT" | |
| echo "SHA=$SHA" >> "$GITHUB_OUTPUT" | |
| set_pending_status: | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| permissions: 'write-all' | |
| needs: | |
| - 'parse_run_context' | |
| if: "github.repository == 'google-gemini/gemini-cli' && always()" | |
| steps: | |
| - name: 'Set pending status' | |
| uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master | |
| if: "github.repository == 'google-gemini/gemini-cli' && always()" | |
| with: | |
| allowForks: 'true' | |
| repo: '${{ github.repository }}' | |
| sha: '${{ needs.parse_run_context.outputs.sha }}' | |
| token: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}' | |
| status: 'pending' | |
| context: 'E2E (Chained)' | |
| e2e_linux: | |
| name: 'E2E Test (Linux) - ${{ matrix.sandbox }}' | |
| needs: | |
| - 'merge_queue_skipper' | |
| - 'parse_run_context' | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| if: | | |
| github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| sandbox: | |
| - 'sandbox:none' | |
| - 'sandbox:docker' | |
| node-version: | |
| - '20.x' | |
| steps: | |
| - name: 'Checkout' | |
| uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 | |
| with: | |
| ref: '${{ needs.parse_run_context.outputs.sha }}' | |
| repository: '${{ needs.parse_run_context.outputs.repository }}' | |
| - name: 'Set up Node.js ${{ matrix.node-version }}' | |
| uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 | |
| with: | |
| node-version: '${{ matrix.node-version }}' | |
| - name: 'Install dependencies' | |
| run: 'npm ci' | |
| - name: 'Build project' | |
| run: 'npm run build' | |
| - name: 'Set up Docker' | |
| if: "${{matrix.sandbox == 'sandbox:docker'}}" | |
| uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3 | |
| - name: 'Run E2E tests' | |
| env: | |
| GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' | |
| KEEP_OUTPUT: 'true' | |
| VERBOSE: 'true' | |
| BUILD_SANDBOX_FLAGS: '--cache-from type=gha --cache-to type=gha,mode=max' | |
| shell: 'bash' | |
| run: | | |
| if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then | |
| npm run test:integration:sandbox:docker | |
| else | |
| npm run test:integration:sandbox:none | |
| fi | |
| e2e_mac: | |
| name: 'E2E Test (macOS)' | |
| needs: | |
| - 'merge_queue_skipper' | |
| - 'parse_run_context' | |
| runs-on: 'macos-latest' | |
| if: | | |
| github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true') | |
| steps: | |
| - name: 'Checkout' | |
| uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 | |
| with: | |
| ref: '${{ needs.parse_run_context.outputs.sha }}' | |
| repository: '${{ needs.parse_run_context.outputs.repository }}' | |
| - name: 'Set up Node.js 20.x' | |
| uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 | |
| with: | |
| node-version: '20.x' | |
| - name: 'Install dependencies' | |
| run: 'npm ci' | |
| - name: 'Build project' | |
| run: 'npm run build' | |
| - name: 'Fix rollup optional dependencies on macOS' | |
| if: "${{runner.os == 'macOS'}}" | |
| run: | | |
| npm cache clean --force | |
| - name: 'Run E2E tests (non-Windows)' | |
| if: "${{runner.os != 'Windows'}}" | |
| env: | |
| GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' | |
| KEEP_OUTPUT: 'true' | |
| SANDBOX: 'sandbox:none' | |
| VERBOSE: 'true' | |
| run: 'npm run test:integration:sandbox:none' | |
| e2e_windows: | |
| name: 'Slow E2E - Win' | |
| needs: | |
| - 'merge_queue_skipper' | |
| - 'parse_run_context' | |
| if: | | |
| github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true') | |
| runs-on: 'gemini-cli-windows-16-core' | |
| steps: | |
| - name: 'Checkout' | |
| uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 | |
| with: | |
| ref: '${{ needs.parse_run_context.outputs.sha }}' | |
| repository: '${{ needs.parse_run_context.outputs.repository }}' | |
| - name: 'Set up Node.js 20.x' | |
| uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 | |
| with: | |
| node-version: '20.x' | |
| cache: 'npm' | |
| - name: 'Configure Windows Defender exclusions' | |
| run: | | |
| Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force | |
| Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force | |
| Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force | |
| Add-MpPreference -ExclusionPath "$env:TEMP" -Force | |
| shell: 'pwsh' | |
| - name: 'Configure npm for Windows performance' | |
| run: | | |
| npm config set progress false | |
| npm config set audit false | |
| npm config set fund false | |
| npm config set loglevel error | |
| npm config set maxsockets 32 | |
| npm config set registry https://registry.npmjs.org/ | |
| shell: 'pwsh' | |
| - name: 'Install dependencies' | |
| run: 'npm ci' | |
| shell: 'pwsh' | |
| - name: 'Build project' | |
| run: 'npm run build' | |
| shell: 'pwsh' | |
| - name: 'Ensure Chrome is available' | |
| shell: 'pwsh' | |
| run: | | |
| $chromePaths = @( | |
| "${env:ProgramFiles}\Google\Chrome\Application\chrome.exe", | |
| "${env:ProgramFiles(x86)}\Google\Chrome\Application\chrome.exe" | |
| ) | |
| $chromeExists = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1 | |
| if (-not $chromeExists) { | |
| Write-Host 'Chrome not found, installing via Chocolatey...' | |
| choco install googlechrome -y --no-progress --ignore-checksums | |
| } | |
| $installed = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1 | |
| if ($installed) { | |
| Write-Host "Chrome found at: $installed" | |
| & $installed --version | |
| } else { | |
| Write-Error 'Chrome installation failed' | |
| exit 1 | |
| } | |
| - name: 'Run E2E tests' | |
| env: | |
| GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' | |
| KEEP_OUTPUT: 'true' | |
| SANDBOX: 'sandbox:none' | |
| VERBOSE: 'true' | |
| NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256' | |
| UV_THREADPOOL_SIZE: '32' | |
| NODE_ENV: 'test' | |
| shell: 'pwsh' | |
| run: 'npm run test:integration:sandbox:none' | |
| evals: | |
| name: 'Evals (ALWAYS_PASSING)' | |
| needs: | |
| - 'merge_queue_skipper' | |
| - 'parse_run_context' | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| if: | | |
| github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true') | |
| steps: | |
| - name: 'Checkout' | |
| uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 | |
| with: | |
| ref: '${{ needs.parse_run_context.outputs.sha }}' | |
| repository: '${{ needs.parse_run_context.outputs.repository }}' | |
| fetch-depth: 0 | |
| - name: 'Set up Node.js 20.x' | |
| uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 | |
| with: | |
| node-version: '20.x' | |
| - name: 'Install dependencies' | |
| run: 'npm ci' | |
| - name: 'Build project' | |
| run: 'npm run build' | |
| - name: 'Check if evals should run' | |
| id: 'check_evals' | |
| run: | | |
| SHOULD_RUN=$(node scripts/changed_prompt.js) | |
| echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" | |
| - name: 'Run Evals (Required to pass)' | |
| if: "${{ steps.check_evals.outputs.should_run == 'true' }}" | |
| env: | |
| GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' | |
| GEMINI_MODEL: 'gemini-3-pro-preview' | |
| # Disable Vitest internal retries to avoid double-retrying; | |
| # custom retry logic is handled in evals/test-helper.ts | |
| VITEST_RETRY: 0 | |
| run: 'npm run test:always_passing_evals' | |
| - name: 'Upload Reliability Logs' | |
| if: "always() && steps.check_evals.outputs.should_run == 'true'" | |
| uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4 | |
| with: | |
| name: 'eval-logs-${{ github.run_id }}-${{ github.run_attempt }}' | |
| path: 'evals/logs/api-reliability.jsonl' | |
| retention-days: 7 | |
| e2e: | |
| name: 'E2E' | |
| if: | | |
| github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true') | |
| needs: | |
| - 'e2e_linux' | |
| - 'e2e_mac' | |
| - 'e2e_windows' | |
| - 'evals' | |
| - 'merge_queue_skipper' | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| steps: | |
| - name: 'Check E2E test results' | |
| run: | | |
| if [[ ${NEEDS_E2E_LINUX_RESULT} != 'success' || \ | |
| ${NEEDS_E2E_MAC_RESULT} != 'success' || \ | |
| ${NEEDS_E2E_WINDOWS_RESULT} != 'success' || \ | |
| ${NEEDS_EVALS_RESULT} != 'success' ]]; then | |
| echo "One or more E2E jobs failed." | |
| exit 1 | |
| fi | |
| echo "All required E2E jobs passed!" | |
| env: | |
| NEEDS_E2E_LINUX_RESULT: '${{ needs.e2e_linux.result }}' | |
| NEEDS_E2E_MAC_RESULT: '${{ needs.e2e_mac.result }}' | |
| NEEDS_E2E_WINDOWS_RESULT: '${{ needs.e2e_windows.result }}' | |
| NEEDS_EVALS_RESULT: '${{ needs.evals.result }}' | |
| set_workflow_status: | |
| runs-on: 'gemini-cli-ubuntu-16-core' | |
| permissions: 'write-all' | |
| if: "github.repository == 'google-gemini/gemini-cli' && always()" | |
| needs: | |
| - 'parse_run_context' | |
| - 'e2e' | |
| steps: | |
| - name: 'Set workflow status' | |
| uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master | |
| if: "github.repository == 'google-gemini/gemini-cli' && always()" | |
| with: | |
| allowForks: 'true' | |
| repo: '${{ github.repository }}' | |
| sha: '${{ needs.parse_run_context.outputs.sha }}' | |
| token: '${{ secrets.GITHUB_TOKEN }}' | |
| status: '${{ needs.e2e.result }}' | |
| context: 'E2E (Chained)' |