E2E #1309
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Trusted workflow: runs in the base repo context via workflow_run so self-hosted | |
| # runners and the workflow YAML are never exposed to untrusted fork code. | |
| # Untrusted code (binaries + pytest) runs ONLY inside disposable driver VMs. | |
| # | |
| # Trigger migration to pull_request_target is deferred to a follow-up PR. | |
| name: E2E | |
| on: | |
| workflow_run: | |
| workflows: ["CI"] | |
| types: [completed] | |
| permissions: | |
| statuses: write | |
| actions: read | |
| contents: read | |
| jobs: | |
| mark-skipped: | |
| name: Mark E2E skipped | |
| runs-on: ubuntu-latest | |
| if: github.event.workflow_run.conclusion != 'success' | |
| permissions: | |
| statuses: write | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| sparse-checkout: .github/workflows/e2e.yml | |
| sparse-checkout-cone-mode: false | |
| - name: Mark E2E statuses | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const text = fs.readFileSync('.github/workflows/e2e.yml', 'utf8'); | |
| const matches = [...text.matchAll(/^\s*STATUS_CONTEXT:\s*["']([^"']+)["']\s*$/gm)]; | |
| const contexts = [...new Set(matches.map((m) => m[1]))]; | |
| if (contexts.length === 0) { | |
| core.setFailed('No STATUS_CONTEXT values found in .github/workflows/e2e.yml'); | |
| return; | |
| } | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| const conclusion = '${{ github.event.workflow_run.conclusion }}'; | |
| const desc = conclusion === 'failure' | |
| ? 'Skipped: CI did not pass' | |
| : `Skipped: CI ${conclusion}`; | |
| for (const ctx of contexts) { | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, repo: context.repo.repo, sha, | |
| state: 'failure', | |
| context: ctx, | |
| description: desc, | |
| target_url: '${{ github.event.workflow_run.html_url }}' | |
| }); | |
| } | |
| e2e: | |
| name: ${{ matrix.STATUS_CONTEXT }} | |
| runs-on: [self-hosted, spur-e2e, gpu] | |
| if: github.event.workflow_run.conclusion == 'success' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - cluster_type: native-host | |
| STATUS_CONTEXT: "E2E / native-host" | |
| - cluster_type: k8s | |
| STATUS_CONTEXT: "E2E / k8s" | |
| env: | |
| STATUS_CONTEXT: ${{ matrix.STATUS_CONTEXT }} | |
| # LogLevel=ERROR silences the known-hosts warning from UserKnownHostsFile=/dev/null. | |
| SSH_OPTS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" | |
| RUN_DIR: /tmp/spur-e2e-${{ github.run_id }} | |
| steps: | |
| - name: Set pending status | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| sha: '${{ github.event.workflow_run.head_sha }}', | |
| state: 'pending', | |
| target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', | |
| context: '${{ env.STATUS_CONTEXT }}' | |
| }); | |
| - name: Download release binaries | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: release-binaries | |
| path: ${{ env.RUN_DIR }}/release-binaries | |
| run-id: ${{ github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download E2E assets | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: e2e-assets | |
| path: ${{ env.RUN_DIR }}/e2e-assets | |
| run-id: ${{ github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download container image (k8s only) | |
| if: matrix.cluster_type == 'k8s' | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: spur-image | |
| path: ${{ env.RUN_DIR }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # --- Host-side: ONLY /opt/spur-ci/bin scripts from here --- | |
| - name: Generate ephemeral SSH keypair | |
| id: ssh | |
| run: | | |
| KEY_DIR=$(mktemp -d /tmp/spur-ci-keys-XXXXXX) | |
| ssh-keygen -t ed25519 -f "$KEY_DIR/id_ed25519" -N "" -q | |
| echo "key_dir=$KEY_DIR" >> "$GITHUB_OUTPUT" | |
| echo "pubkey=$KEY_DIR/id_ed25519.pub" >> "$GITHUB_OUTPUT" | |
| echo "privkey=$KEY_DIR/id_ed25519" >> "$GITHUB_OUTPUT" | |
| - name: Select image for cluster type | |
| id: image | |
| run: | | |
| if [[ "${{ matrix.cluster_type }}" == "k8s" ]]; then | |
| echo "path=${SPUR_CI_IMAGE_DIR}/spur-ci-k8s.qcow2" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "path=${SPUR_CI_IMAGE_DIR}/spur-ci-base.qcow2" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Bring up ephemeral VM cluster | |
| id: cluster | |
| run: | | |
| CLUSTER_ENV=$(mktemp /tmp/cluster-env-XXXXXX.sh) | |
| echo "cluster_env=$CLUSTER_ENV" >> "$GITHUB_OUTPUT" | |
| /opt/spur-ci/bin/cluster-up.sh \ | |
| --prefix "${SPUR_CI_VM_PREFIX}-${{ github.run_id }}" \ | |
| --count 4 \ | |
| --image "${{ steps.image.outputs.path }}" \ | |
| --gpus-per-vm 1 \ | |
| --skip-gpu-for 0 \ | |
| --ssh-pubkey "${{ steps.ssh.outputs.pubkey }}" \ | |
| --network "${SPUR_CI_NETWORK}" \ | |
| --gpu-vfs "${SPUR_CI_GPU_VFS}" \ | |
| > "$CLUSTER_ENV" | |
| source "$CLUSTER_ENV" | |
| echo "driver_ip=${VM_0_IP}" >> "$GITHUB_OUTPUT" | |
| echo "gpu_ips=${VM_1_IP},${VM_2_IP},${VM_3_IP}" >> "$GITHUB_OUTPUT" | |
| - name: Wait for VMs to be reachable | |
| run: | | |
| source "${{ steps.cluster.outputs.cluster_env }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| for ip in $VM_0_IP $VM_1_IP $VM_2_IP $VM_3_IP; do | |
| echo "Waiting for $ip..." | |
| ok=false | |
| for i in $(seq 1 90); do | |
| if ssh $SSH_OPTS -o ConnectTimeout=3 -o BatchMode=yes \ | |
| -i "$KEY" ci@"$ip" true 2>/dev/null; then | |
| echo " $ip ready" | |
| ok=true | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| if [[ "$ok" != "true" ]]; then | |
| echo "ERROR: $ip not reachable after 180s" >&2 | |
| exit 1 | |
| fi | |
| done | |
| - name: Bootstrap k8s cluster (k8s only) | |
| if: matrix.cluster_type == 'k8s' | |
| id: k8s-bootstrap | |
| run: | | |
| source "${{ steps.cluster.outputs.cluster_env }}" | |
| KUBECONFIG_DIR=$(mktemp -d) | |
| /opt/spur-ci/bin/k8s-bootstrap.sh \ | |
| --control-plane "$VM_0_IP" \ | |
| --workers "$VM_1_IP,$VM_2_IP,$VM_3_IP" \ | |
| --ssh-key "${{ steps.ssh.outputs.privkey }}" \ | |
| --ssh-user ci \ | |
| --kubeconfig "$KUBECONFIG_DIR/config" \ | |
| --image-tar "${{ env.RUN_DIR }}/spur-image.tar" | |
| echo "kubeconfig=$KUBECONFIG_DIR/config" >> "$GITHUB_OUTPUT" | |
| - name: Push artifacts into driver VM | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" "mkdir -p ${{ env.RUN_DIR }}" | |
| scp $SSH_OPTS -i "$KEY" -r "${{ env.RUN_DIR }}/release-binaries" ci@"$DRIVER_IP":"${{ env.RUN_DIR }}/" | |
| scp $SSH_OPTS -i "$KEY" -r "${{ env.RUN_DIR }}/e2e-assets" ci@"$DRIVER_IP":"${{ env.RUN_DIR }}/" | |
| scp $SSH_OPTS -i "$KEY" "$KEY" ci@"$DRIVER_IP":"${{ env.RUN_DIR }}/ci-ssh-key" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" "chmod 600 ${{ env.RUN_DIR }}/ci-ssh-key" | |
| - name: Load AppArmor profile for rootless containers | |
| if: matrix.cluster_type == 'native-host' | |
| run: | | |
| GPU_IPS="${{ steps.cluster.outputs.gpu_ips }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| SPURD="/tmp/spur-ci-bin-${{ github.run_id }}/spurd" | |
| PROF="abi <abi/4.0>, | |
| profile spur-ci ${SPURD} flags=(unconfined) { | |
| userns, | |
| }" | |
| IFS=',' read -ra NODES <<< "$GPU_IPS" | |
| for node in "${NODES[@]}"; do | |
| ssh $SSH_OPTS -i "$KEY" ci@"$node" "echo '${PROF}' | sudo apparmor_parser -r 2>/dev/null || true" | |
| done | |
| - name: Run tests inside driver VM (native-host) | |
| if: matrix.cluster_type == 'native-host' | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| GPU_IPS="${{ steps.cluster.outputs.gpu_ips }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" bash -s <<REMOTE | |
| set -euo pipefail | |
| export SPUR_TEST_NODES="$GPU_IPS" | |
| export SPUR_TEST_SSH_USER=ci | |
| export SPUR_TEST_SSH_KEY=${{ env.RUN_DIR }}/ci-ssh-key | |
| export SPUR_TEST_BINARIES_DIR=${{ env.RUN_DIR }}/release-binaries | |
| export SPUR_TEST_REMOTE_BIN_DIR=/tmp/spur-ci-bin-${{ github.run_id }} | |
| export SPUR_TEST_GPU_VENV=/opt/spur-ci/gpu-venv | |
| chmod +x ${{ env.RUN_DIR }}/release-binaries/* | |
| source /opt/spur-ci/test-venv/bin/activate | |
| pytest ${{ env.RUN_DIR }}/e2e-assets/tests/native_host/e2e/ -v --junitxml=${{ env.RUN_DIR }}/results.xml | |
| REMOTE | |
| - name: Apply RBAC and verify cluster state (k8s) | |
| if: matrix.cluster_type == 'k8s' | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| NS="spur-ci-${{ github.run_id }}" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" bash -s <<REMOTE | |
| set -euo pipefail | |
| export KUBECONFIG=/home/ci/.kube/config | |
| # Create test namespace | |
| kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - | |
| # Apply RBAC with namespace patched | |
| sed "s/namespace: spur/namespace: $NS/g" \ | |
| ${{ env.RUN_DIR }}/e2e-assets/tests/k8s/e2e/manifests/rbac.yaml | kubectl apply -f - | |
| # Verify nodes are labeled | |
| echo "=== Labeled nodes ===" | |
| kubectl get nodes -l spur.amd.com/managed=true -o wide | |
| REMOTE | |
| - name: Run tests inside driver VM (k8s) | |
| if: matrix.cluster_type == 'k8s' | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| NS="spur-ci-${{ github.run_id }}" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" bash -s <<REMOTE | |
| set -euo pipefail | |
| export KUBECONFIG=/home/ci/.kube/config | |
| export SPUR_CI_IMAGE=docker.io/library/spur:ci | |
| export SPUR_TEST_NS="$NS" | |
| source /opt/spur-ci/test-venv/bin/activate | |
| pytest ${{ env.RUN_DIR }}/e2e-assets/tests/k8s/e2e/ -v --junitxml=${{ env.RUN_DIR }}/results.xml | |
| REMOTE | |
| - name: Dump cluster diagnostics (k8s) | |
| if: failure() && matrix.cluster_type == 'k8s' | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| mkdir -p "${{ env.RUN_DIR }}/e2e-results" | |
| ssh $SSH_OPTS -i "$KEY" ci@"$DRIVER_IP" bash -s <<'REMOTE' || true | |
| set -uo pipefail | |
| export KUBECONFIG=/home/ci/.kube/config | |
| D=/tmp/cluster-dump | |
| rm -rf "$D" && mkdir -p "$D" | |
| kubectl get events -A --sort-by=.lastTimestamp > "$D/events.txt" 2>&1 || true | |
| kubectl get pods -A -o wide > "$D/pods.txt" 2>&1 || true | |
| for ns in $(kubectl get ns -o name 2>/dev/null | sed 's|namespace/||' | grep -E 'spur'); do | |
| for selector in app=spurctld app=spur-k8s-operator; do | |
| for p in $(kubectl -n "$ns" get pods -l "$selector" -o name 2>/dev/null); do | |
| name=${p#pod/} | |
| kubectl -n "$ns" logs "$name" --all-containers --timestamps > "$D/log-$ns-$name.txt" 2>&1 || true | |
| kubectl -n "$ns" logs "$name" --all-containers --previous --timestamps > "$D/log-$ns-$name-prev.txt" 2>&1 || true | |
| done | |
| done | |
| done | |
| REMOTE | |
| scp $SSH_OPTS -i "$KEY" -r ci@"$DRIVER_IP":/tmp/cluster-dump "${{ env.RUN_DIR }}/e2e-results/" 2>/dev/null || true | |
| - name: Collect results from driver VM | |
| if: always() | |
| run: | | |
| DRIVER_IP="${{ steps.cluster.outputs.driver_ip }}" | |
| KEY="${{ steps.ssh.outputs.privkey }}" | |
| mkdir -p "${{ env.RUN_DIR }}/e2e-results" | |
| scp $SSH_OPTS -i "$KEY" \ | |
| ci@"$DRIVER_IP":"${{ env.RUN_DIR }}/results.xml" "${{ env.RUN_DIR }}/e2e-results/" 2>/dev/null || true | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: e2e-results-${{ matrix.cluster_type }} | |
| path: ${{ env.RUN_DIR }}/e2e-results/ | |
| retention-days: 3 | |
| if-no-files-found: ignore | |
| - name: Tear down ephemeral VM cluster | |
| if: always() | |
| run: | | |
| /opt/spur-ci/bin/cluster-down.sh \ | |
| --prefix "${SPUR_CI_VM_PREFIX}-${{ github.run_id }}" \ | |
| --count 4 | |
| - name: Cleanup ephemeral keys | |
| if: always() | |
| run: rm -rf "${{ steps.ssh.outputs.key_dir }}" | |
| - name: Cleanup run directory | |
| if: always() | |
| run: rm -rf "${{ env.RUN_DIR }}" | |
| - name: Report status | |
| if: always() | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| sha: '${{ github.event.workflow_run.head_sha }}', | |
| state: '${{ job.status }}' === 'success' ? 'success' : 'failure', | |
| target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', | |
| context: '${{ env.STATUS_CONTEXT }}', | |
| description: '${{ job.status }}' | |
| }); |