refactor: add platform-neutral TLS startup hooks #5815
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI - OpenShift E2E Tests | |
| # Permissions needed for various jobs | |
| permissions: | |
| contents: read | |
| packages: write | |
| pull-requests: write # For posting comments on PRs | |
| statuses: write # For reporting status on fork PR commits | |
| # Cancel previous runs on the same PR to avoid resource conflicts | |
| # Only group by PR number for legitimate triggers (pull_request, workflow_dispatch, /ok-to-test, or /retest comments) | |
| # Regular comments get a unique group (run_id) so they don't cancel in-progress test runs | |
| # | |
| # Logic: | |
| # - Regular comments (not /ok-to-test or /retest): unique group prevents cancellation of real tests | |
| # - Valid triggers: group 'e2e-openshift-{pr_number}' (can cancel previous runs for same PR) | |
| # - Fallback chain for ID: pull_request.number -> issue.number -> run_id | |
| # | |
| # NOTE: Valid command list (/ok-to-test, /retest) must stay in sync with gate job validation (line ~125) | |
| concurrency: | |
| group: >- | |
| ${{ | |
| github.event_name == 'issue_comment' && | |
| !contains(github.event.comment.body, '/ok-to-test') && | |
| !contains(github.event.comment.body, '/retest') | |
| && format('comment-isolated-{0}', github.run_id) | |
| || format('e2e-openshift-{0}', | |
| github.event.pull_request.number | |
| || github.event.issue.number | |
| || github.run_id) | |
| }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| # Allow maintainers to trigger tests on fork PRs via /ok-to-test comment | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| model_id: | |
| description: 'Model ID' | |
| required: false | |
| default: 'e2ewva/dummy-model' | |
| accelerator_type: | |
| description: 'Accelerator type (H100, A100, L40S)' | |
| required: false | |
| default: 'H100' | |
| request_rate: | |
| description: 'Request rate (req/s)' | |
| required: false | |
| default: '20' | |
| num_prompts: | |
| description: 'Number of prompts' | |
| required: false | |
| default: '3000' | |
| skip_cleanup: | |
| description: 'Skip cleanup after tests' | |
| required: false | |
| default: 'false' | |
| max_num_seqs: | |
| description: 'vLLM max batch size (lower = easier to saturate)' | |
| required: false | |
| default: '1' | |
| hpa_stabilization_seconds: | |
| description: 'HPA stabilization window in seconds' | |
| required: false | |
| default: '240' | |
| jobs: | |
| # Check if PR contains code changes (not just docs/metadata) | |
| check-code-changes: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| has_code_changes: ${{ steps.set-output.outputs.has_code_changes }} | |
| steps: | |
| - name: Checkout source | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v6 | |
| - name: Check for code changes | |
| if: github.event_name == 'pull_request' | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| predicate-quantifier: 'every' | |
| filters: | | |
| code: | |
| - '**' | |
| - '!docs/**' | |
| - '!README.md' | |
| - '!CONTRIBUTING.md' | |
| - '!LICENSE' | |
| - '!OWNERS' | |
| - '!PROJECT' | |
| - name: Set output | |
| id: set-output | |
| run: | | |
| if [ "${{ github.event_name }}" != "pull_request" ]; then | |
| # Always run for issue_comment (/ok-to-test, /retest) and workflow_dispatch | |
| echo "has_code_changes=true" >> $GITHUB_OUTPUT | |
| elif [ -n "${{ steps.filter.outputs.code }}" ]; then | |
| echo "has_code_changes=${{ steps.filter.outputs.code }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_code_changes=true" >> $GITHUB_OUTPUT | |
| fi | |
| # Gate: Check permissions and handle /ok-to-test for fork PRs. | |
| # - Maintainers (write access): Tests run automatically on pull_request. | |
| # - Fork PRs: Gate succeeds (no failure) so the PR does not show a false red check; E2E runs | |
| # only after a maintainer comments /ok-to-test. Branch protection should require the | |
| # "e2e-openshift" job so merge stays blocked until that run passes. | |
| gate: | |
| needs: check-code-changes | |
| if: needs.check-code-changes.outputs.has_code_changes == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.check.outputs.should_run }} | |
| pr_number: ${{ steps.check.outputs.pr_number }} | |
| pr_head_sha: ${{ steps.check.outputs.pr_head_sha }} | |
| is_fork_pr: ${{ steps.check.outputs.is_fork_pr }} | |
| steps: | |
| - name: Check permissions and OpenShift E2E triggers (/ok-to-test, /retest) | |
| id: check | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| // Helper to check if user has write access | |
| async function hasWriteAccess(username) { | |
| try { | |
| const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| username: username | |
| }); | |
| const privilegedRoles = ['admin', 'maintain', 'write']; | |
| return privilegedRoles.includes(permission.permission); | |
| } catch (e) { | |
| console.log(`Could not get permissions for ${username}: ${e.message}`); | |
| return false; | |
| } | |
| } | |
| // Always run for workflow_dispatch | |
| if (context.eventName === 'workflow_dispatch') { | |
| core.setOutput('should_run', 'true'); | |
| core.setOutput('pr_number', ''); | |
| core.setOutput('pr_head_sha', context.sha); | |
| core.setOutput('is_fork_pr', 'false'); | |
| return; | |
| } | |
| // Handle issue_comment event (/ok-to-test or /retest) | |
| if (context.eventName === 'issue_comment') { | |
| const comment = context.payload.comment.body.trim(); | |
| const issue = context.payload.issue; | |
| // Only process /ok-to-test or /retest comments on PRs | |
| if (!issue.pull_request) { | |
| console.log('Comment is not on a PR, skipping'); | |
| core.setOutput('should_run', 'false'); | |
| return; | |
| } | |
| // NOTE: This list must stay in sync with concurrency group logic (lines 23-25) | |
| const validCommands = ['/ok-to-test', '/retest']; | |
| if (!validCommands.includes(comment)) { | |
| console.log(`Comment "${comment}" is not a valid trigger command, skipping`); | |
| core.setOutput('should_run', 'false'); | |
| return; | |
| } | |
| // Check if commenter has write access | |
| const commenter = context.payload.comment.user.login; | |
| const hasAccess = await hasWriteAccess(commenter); | |
| if (!hasAccess) { | |
| console.log(`User ${commenter} does not have write access, ignoring ${comment}`); | |
| core.setOutput('should_run', 'false'); | |
| return; | |
| } | |
| // Get PR details to get head SHA | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: issue.number | |
| }); | |
| // Check if PR is from a fork | |
| const baseRepo = `${context.repo.owner}/${context.repo.repo}`; | |
| const headRepo = pr.head.repo ? pr.head.repo.full_name : baseRepo; | |
| const isFork = headRepo !== baseRepo; | |
| console.log(`${comment} approved by ${commenter} for PR #${issue.number}`); | |
| console.log(`PR head SHA: ${pr.head.sha}`); | |
| console.log(`Is fork PR: ${isFork} (head: ${headRepo}, base: ${baseRepo})`); | |
| core.setOutput('should_run', 'true'); | |
| core.setOutput('pr_number', issue.number.toString()); | |
| core.setOutput('pr_head_sha', pr.head.sha); | |
| core.setOutput('is_fork_pr', isFork ? 'true' : 'false'); | |
| // Add reaction to acknowledge | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'rocket' | |
| }); | |
| // Post comment with link to the e2e workflow run | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const cmdDesc = comment === '/ok-to-test' ? 'approve and run' : 're-run'; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issue.number, | |
| body: `🚀 **OpenShift E2E** — ${cmdDesc} (\`${comment}\`)\n\n[View the OpenShift E2E workflow run](${runUrl})` | |
| }); | |
| return; | |
| } | |
| // Handle pull_request event | |
| const pr = context.payload.pull_request; | |
| const prAuthor = pr.user.login; | |
| const prNumber = pr.number; | |
| const prHeadSha = pr.head.sha; | |
| // Check if PR is from a fork | |
| const baseRepo = `${context.repo.owner}/${context.repo.repo}`; | |
| const headRepo = pr.head.repo ? pr.head.repo.full_name : baseRepo; | |
| const isFork = headRepo !== baseRepo; | |
| console.log(`PR #${prNumber} is from fork: ${isFork} (head: ${headRepo}, base: ${baseRepo})`); | |
| core.setOutput('pr_number', prNumber.toString()); | |
| core.setOutput('pr_head_sha', prHeadSha); | |
| core.setOutput('is_fork_pr', isFork ? 'true' : 'false'); | |
| // Check if PR author has write access | |
| const isPrivileged = await hasWriteAccess(prAuthor); | |
| console.log(`PR #${prNumber} author ${prAuthor}: privileged=${isPrivileged}`); | |
| // Check if we already posted a bot comment | |
| const comments = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber | |
| }); | |
| const botComment = comments.data.find(c => | |
| c.user.type === 'Bot' && | |
| c.body.includes('ok-to-test') | |
| ); | |
| // Helper to safely post a comment (may fail on fork PRs due to permissions) | |
| async function tryPostComment(body) { | |
| try { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body: body | |
| }); | |
| return true; | |
| } catch (e) { | |
| // Fork PRs can't post comments on pull_request event (GitHub security restriction) | |
| console.log(`Could not post comment (expected for fork PRs): ${e.message}`); | |
| return false; | |
| } | |
| } | |
| if (isPrivileged) { | |
| // For maintainer/admin fork PRs, we need to trigger via /ok-to-test | |
| // because fork PRs don't have access to secrets on pull_request event | |
| if (isFork) { | |
| console.log(`Maintainer fork PR detected - auto-triggering /ok-to-test for ${prAuthor}`); | |
| core.setOutput('should_run', 'false'); // Don't run on pull_request event | |
| // Auto-post /ok-to-test to trigger issue_comment workflow | |
| if (!botComment) { | |
| const posted = await tryPostComment(`/ok-to-test`); | |
| if (!posted) { | |
| console.log('Note: Maintainer will need to manually comment /ok-to-test'); | |
| } | |
| } | |
| // Do not fail the gate: fork PRs cannot run E2E on pull_request (no secrets). | |
| // Gate succeeds so the PR does not show a false failure; branch protection | |
| // should require "e2e-openshift" so merge stays blocked until /ok-to-test run passes. | |
| return; | |
| } | |
| // Non-fork PR from maintainer - run directly | |
| core.setOutput('should_run', 'true'); | |
| return; | |
| } | |
| // External contributor - post instructions and skip | |
| console.log('External contributor PR - posting instructions'); | |
| core.setOutput('should_run', 'false'); | |
| if (!botComment) { | |
| const posted = await tryPostComment(`👋 Thanks for your contribution!\n\nThis PR is from a fork, so **OpenShift E2E** (GPU) tests require approval to run.\n\n**For maintainers/admins:** Comment \`/ok-to-test\` to approve and trigger **OpenShift E2E** on this PR, or \`/retest\` to re-run OpenShift E2E (e.g. after a failure or new commits).\n\n**For contributors:** Please wait for a maintainer or admin to approve running the tests.`); | |
| if (!posted) { | |
| console.log('Note: Could not post instructions comment on fork PR'); | |
| } | |
| } | |
| // Do not fail the gate: GitHub does not allow updating status from upstream on fork | |
| // PRs, so a failed gate would stay red even after /ok-to-test run passes. Let the gate | |
| // succeed; branch protection should require "e2e-openshift" so merge stays blocked | |
| // until a maintainer comments /ok-to-test and E2E passes. | |
| - name: Write workflow summary | |
| if: always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const shouldRun = '${{ steps.check.outputs.should_run }}'; | |
| const isFork = '${{ steps.check.outputs.is_fork_pr }}'; | |
| const eventName = '${{ github.event_name }}'; | |
| if (shouldRun === 'true') { | |
| core.summary.addRaw('✅ **E2E tests will run** for this trigger.\n').write(); | |
| } else if (isFork === 'true' && eventName === 'pull_request') { | |
| core.summary.addRaw([ | |
| '⏸️ **E2E tests skipped — fork PR**\n\n', | |
| 'Fork PRs cannot run E2E on `pull_request` events (no access to secrets/GPU runners).\n\n', | |
| 'A maintainer must comment \`/ok-to-test\` to trigger the **OpenShift E2E** suite. ', | |
| 'Branch protection should require **e2e-openshift** so merge stays blocked until E2E passes.\n', | |
| ].join('')).write(); | |
| } else { | |
| core.summary.addRaw('⏸️ **E2E tests were skipped** (gate check did not pass for this trigger).\n').write(); | |
| } | |
| # Build the WVA controller image on GitHub-hosted runner (has proper Docker setup) | |
| # Note: Skip for fork PRs on pull_request event (no secrets access). | |
| # For fork PRs, build-image runs via issue_comment trigger (/ok-to-test). | |
| build-image: | |
| needs: gate | |
| if: | | |
| needs.gate.outputs.should_run == 'true' && | |
| (needs.gate.outputs.is_fork_pr != 'true' || github.event_name != 'pull_request') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.build.outputs.image_tag }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v6 | |
| with: | |
| # Use PR head SHA from gate (works for both pull_request and issue_comment) | |
| ref: ${{ needs.gate.outputs.pr_head_sha }} | |
| fetch-depth: 0 | |
| - name: Merge main into PR branch | |
| if: github.event_name == 'issue_comment' | |
| run: | | |
| git config user.email "ci@github.com" | |
| git config user.name "CI" | |
| git remote add upstream ${{ github.server_url }}/${{ github.repository }}.git | |
| git fetch upstream main | |
| git merge upstream/main --no-edit | |
| - name: Log in to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.CR_USER }} | |
| password: ${{ secrets.CR_TOKEN }} | |
| - name: Build and push image | |
| id: build | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| # Use PR head SHA from gate | |
| GIT_REF: ${{ needs.gate.outputs.pr_head_sha }} | |
| run: deploy/ci-e2e-openshift/build-push-image.sh | |
| # Run e2e tests on OpenShift self-hosted runner (vllm-d cluster). | |
| # pok-prod runners are reserved for nightly E2E only. | |
| e2e-openshift: | |
| runs-on: [self-hosted, openshift, vllm-d] | |
| needs: [gate, build-image] | |
| if: needs.gate.outputs.should_run == 'true' | |
| env: | |
| MODEL_ID: ${{ github.event.inputs.model_id || 'e2ewva/dummy-model' }} | |
| GOTOOLCHAIN: auto | |
| ACCELERATOR_TYPE: ${{ github.event.inputs.accelerator_type || 'A100' }} | |
| REQUEST_RATE: ${{ github.event.inputs.request_rate || '20' }} | |
| NUM_PROMPTS: ${{ github.event.inputs.num_prompts || '3000' }} | |
| MAX_NUM_SEQS: ${{ github.event.inputs.max_num_seqs || '5' }} | |
| HPA_STABILIZATION_SECONDS: ${{ github.event.inputs.hpa_stabilization_seconds || '240' }} | |
| SKIP_CLEANUP: ${{ github.event.inputs.skip_cleanup || 'false' }} | |
| LLM_D_ROUTER_VERSION: v0.9.0 | |
| GAIE_VERSION: v1.5.0 | |
| # PR-specific namespaces for isolation between concurrent PR tests | |
| LLMD_NAMESPACE: llm-d-autoscaler-pr-${{ needs.gate.outputs.pr_number || github.run_id }} | |
| WVA_NAMESPACE: llm-d-autoscaler-pr-${{ needs.gate.outputs.pr_number || github.run_id }} | |
| # Unique release names per run to avoid conflicts | |
| WVA_RELEASE_NAME: wva-e2e-${{ github.run_id }} | |
| MODEL_A1_RELEASE: model-a1-${{ github.run_id }} | |
| # Use the image built in the previous job | |
| WVA_IMAGE_TAG: ${{ needs.build-image.outputs.image_tag }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v6 | |
| with: | |
| # Use PR head SHA from gate (works for both pull_request and issue_comment) | |
| ref: ${{ needs.gate.outputs.pr_head_sha }} | |
| fetch-depth: 0 | |
| - name: Merge main into PR branch | |
| if: github.event_name == 'issue_comment' | |
| run: | | |
| git config user.email "ci@github.com" | |
| git config user.name "CI" | |
| git remote add upstream ${{ github.server_url }}/${{ github.repository }}.git | |
| git fetch upstream main | |
| git merge upstream/main --no-edit | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: go.mod | |
| cache-dependency-path: ./go.sum | |
| - name: Verify Go toolchain | |
| run: deploy/ci-e2e-openshift/verify-go-toolchain.sh | |
| - name: Install tools (kubectl, oc, helm, make, jq, yq) | |
| run: deploy/ci-e2e-openshift/install-tools.sh | |
| - name: Verify cluster access | |
| run: deploy/ci-e2e-openshift/verify-cluster-access.sh | |
| - name: Verify correct cluster (vllm-d, not pok-prod) | |
| run: deploy/ci-e2e-openshift/verify-correct-cluster.sh | |
| - name: Check GPU availability | |
| id: gpu-check | |
| run: deploy/ci-e2e-openshift/check-gpu-availability.sh | |
| - name: Post GPU status to PR | |
| if: always() && needs.gate.outputs.pr_number != '' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ needs.gate.outputs.pr_number }} | |
| run: | | |
| GPU_STATUS="${{ steps.gpu-check.outcome }}" | |
| GPU_AVAIL="${{ steps.gpu-check.outputs.gpu_available }}" | |
| TOTAL_GPUS="${{ steps.gpu-check.outputs.total_gpus }}" | |
| ALLOCATED_GPUS="${{ steps.gpu-check.outputs.allocated_gpus }}" | |
| AVAILABLE_GPUS="${{ steps.gpu-check.outputs.available_gpus }}" | |
| TOTAL_CPU="${{ steps.gpu-check.outputs.total_cpu }}" | |
| TOTAL_MEM_GI="${{ steps.gpu-check.outputs.total_mem_gi }}" | |
| NODE_COUNT="${{ steps.gpu-check.outputs.node_count }}" | |
| GPU_NODE_COUNT="${{ steps.gpu-check.outputs.gpu_node_count }}" | |
| REQUIRED_GPUS="${{ steps.gpu-check.outputs.required_gpus }}" | |
| RECOMMENDED_GPUS="${{ steps.gpu-check.outputs.recommended_gpus }}" | |
| NL=$'\n' | |
| TABLE="| Resource | Total | Allocated | Available |${NL}|----------|-------|-----------|----------|${NL}| GPUs | $TOTAL_GPUS | $ALLOCATED_GPUS | **$AVAILABLE_GPUS** |${NL}${NL}| Cluster | Value |${NL}|---------|-------|${NL}| Nodes | $NODE_COUNT ($GPU_NODE_COUNT with GPUs) |${NL}| Total CPU | ${TOTAL_CPU} cores |${NL}| Total Memory | ${TOTAL_MEM_GI} Gi |${NL}| GPUs required | $REQUIRED_GPUS (min) / $RECOMMENDED_GPUS (recommended) |" | |
| if [ "$GPU_STATUS" = "failure" ]; then | |
| HEADER="### GPU Pre-flight Check ❌" | |
| MSG="**Insufficient GPUs** to run OpenShift E2E. Re-run with \`/retest\` (OpenShift E2E) when GPUs free up." | |
| elif [ "$GPU_AVAIL" = "true" ]; then | |
| HEADER="### GPU Pre-flight Check ✅" | |
| MSG="GPUs are available for e2e-openshift tests. Proceeding with deployment." | |
| else | |
| HEADER="### GPU Pre-flight Check ⚠️" | |
| MSG="Low GPU headroom — tests may fail during scale-up phases." | |
| fi | |
| BODY="${HEADER}${NL}${MSG}${NL}${NL}${TABLE}" | |
| PAYLOAD=$(jq -n --arg body "$BODY" '{"body": $body}') | |
| curl -s -X POST \ | |
| -H "Authorization: token $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues/$PR_NUMBER/comments" \ | |
| -d "$PAYLOAD" | |
| - name: Get HF token from cluster secret | |
| id: hf-token | |
| run: deploy/ci-e2e-openshift/get-hf-token.sh | |
| - name: Clean up resources for this PR | |
| run: deploy/ci-e2e-openshift/cleanup-pr-resources.sh | |
| - name: Deploy WVA and llm-d infrastructure | |
| env: | |
| # HF_TOKEN is inherited from GITHUB_ENV (set in 'Get HF token from cluster secret' step) | |
| ENVIRONMENT: openshift | |
| SCALER_BACKEND: keda | |
| LLMD_NS: ${{ env.LLMD_NAMESPACE }} | |
| WVA_NS: ${{ env.WVA_NAMESPACE }} | |
| CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }} | |
| MONITORING_NAMESPACE: openshift-user-workload-monitoring | |
| WVA_METRICS_SECURE: "false" | |
| ENABLE_SCALE_TO_ZERO: "true" | |
| KV_SPARE_TRIGGER: "0.5" | |
| QUEUE_SPARE_TRIGGER: "4.5" | |
| VLLM_SVC_PORT: "8000" | |
| ACCELERATOR_TYPE: ${{ env.ACCELERATOR_TYPE }} | |
| GAIE_VERSION: ${{ env.GAIE_VERSION }} | |
| run: deploy/ci-e2e-openshift/deploy-infrastructure.sh | |
| - name: Label namespaces for OpenShift monitoring | |
| run: deploy/ci-e2e-openshift/label-namespaces.sh | |
| - name: Wait for infrastructure to be ready | |
| run: deploy/ci-e2e-openshift/wait-infrastructure-ready.sh | |
| - name: Verify deployment | |
| run: deploy/ci-e2e-openshift/verify-deployment.sh | |
| - name: Verify metrics pipeline | |
| run: deploy/ci-e2e-openshift/verify-metrics-pipeline.sh | |
| - name: Install Go dependencies | |
| run: deploy/ci-e2e-openshift/install-go-dependencies.sh | |
| - name: Run OpenShift E2E tests | |
| env: | |
| # Consolidated e2e test environment variables | |
| ENVIRONMENT: openshift | |
| # Temporary/ To do: real vLLM decode scheduling was flaky on shared GPU runners; tests use llm-d-inference-sim. | |
| USE_SIMULATOR: "true" | |
| SCALE_TO_ZERO_ENABLED: "true" | |
| DEPLOY_LWS: "true" | |
| WVA_NAMESPACE: ${{ env.WVA_NAMESPACE }} | |
| MONITORING_NAMESPACE: openshift-user-workload-monitoring | |
| LLMD_NAMESPACE: ${{ env.LLMD_NAMESPACE }} | |
| # Legacy variables for backward compatibility (if needed by tests) | |
| CONTROLLER_NAMESPACE: ${{ env.WVA_NAMESPACE }} | |
| GATEWAY_NAME: optimized-baseline-inference-gateway-istio | |
| # Pass WVA_RELEASE_NAME so test can filter for current run's resources | |
| WVA_RELEASE_NAME: ${{ env.WVA_RELEASE_NAME }} | |
| # Controller instance label must match what the controller was deployed with | |
| CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }} | |
| MODEL_ID: ${{ env.MODEL_ID }} | |
| REQUEST_RATE: ${{ env.REQUEST_RATE }} | |
| NUM_PROMPTS: ${{ env.NUM_PROMPTS }} | |
| run: deploy/ci-e2e-openshift/run-e2e-tests.sh | |
| - name: Cleanup infrastructure | |
| # Cleanup on success or cancellation, but NOT on failure (preserve for debugging) | |
| # Use SKIP_CLEANUP=true to keep resources after successful runs | |
| if: (success() || cancelled()) && env.SKIP_CLEANUP != 'true' | |
| run: deploy/ci-e2e-openshift/cleanup-infrastructure.sh | |
| - name: Collect cluster diagnostics | |
| if: always() | |
| run: deploy/ci-e2e-openshift/collect-diagnostics.sh | |
| - name: Upload cluster diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: cluster-diagnostics-${{ github.run_id }} | |
| path: /tmp/cluster-diagnostics/ | |
| retention-days: 7 | |
| - name: Scale down GPU workloads on failure | |
| # On failure, scale down decode deployments to free GPUs while preserving | |
| # other resources (VA, HPA, controller, gateway) for debugging | |
| if: failure() | |
| run: deploy/ci-e2e-openshift/scale-down-on-failure.sh | |
| # Report status back to PR for issue_comment triggered runs | |
| # This ensures fork PRs show the correct status after /ok-to-test runs complete | |
| report-status: | |
| runs-on: ubuntu-latest | |
| needs: [gate, e2e-openshift] | |
| # Run always (even on failure) but only for issue_comment events | |
| if: always() && github.event_name == 'issue_comment' && needs.gate.outputs.should_run == 'true' | |
| steps: | |
| - name: Report status to PR | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const prHeadSha = '${{ needs.gate.outputs.pr_head_sha }}'; | |
| const e2eResult = '${{ needs.e2e-openshift.result }}'; | |
| // Map job result to commit status | |
| let state, description; | |
| if (e2eResult === 'success') { | |
| state = 'success'; | |
| description = 'E2E tests passed'; | |
| } else if (e2eResult === 'skipped') { | |
| state = 'pending'; | |
| description = 'E2E tests skipped'; | |
| } else if (e2eResult === 'cancelled') { | |
| state = 'failure'; | |
| description = 'E2E tests cancelled'; | |
| } else { | |
| state = 'failure'; | |
| description = 'E2E tests failed'; | |
| } | |
| console.log(`Reporting status to PR commit ${prHeadSha}: ${state} - ${description}`); | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: prHeadSha, | |
| state: state, | |
| target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| description: description, | |
| context: '${{ github.workflow }} / e2e (comment trigger)' | |
| }); | |
| console.log('Status reported successfully'); |