fix(spurctld): remove dead code and wire leader-forwarding fallback (… #286
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| build-and-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install protobuf compiler | |
| run: sudo apt-get install -y protobuf-compiler | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cache cargo registry and build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Build | |
| run: cargo build --all-targets | |
| - name: Test | |
| run: cargo test | |
| fmt: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt | |
| - name: Check formatting | |
| run: cargo fmt --all --check | |
| clippy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install protobuf compiler | |
| run: sudo apt-get install -y protobuf-compiler | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: clippy | |
| - name: Cache cargo registry and build | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-clippy- | |
| - name: Clippy | |
| run: cargo clippy --workspace --exclude spur-ffi --all-targets -- -W clippy::all | |
| cluster: | |
| name: Cluster Tests (2x MI300X) | |
| runs-on: [self-hosted, mi300x, primary] | |
| needs: [build-and-test, fmt, clippy] | |
| # Only one cluster run at a time; cancel older runs on the same ref. | |
| concurrency: | |
| group: cluster-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build release binaries | |
| run: | | |
| export PATH="$HOME/bin:$HOME/.local/bin:$HOME/.cargo/bin:$PATH" | |
| export PROTOC="$HOME/bin/protoc" | |
| export PROTOC_INCLUDE="$HOME/protoc-install/include" | |
| cargo build --release -j 4 | |
| - name: Stop any running cluster and clear state | |
| run: | | |
| ssh mi300-2 '~/spur/etc/stop-agent.sh' 2>/dev/null || true | |
| ~/spur/etc/stop-all.sh 2>/dev/null || true | |
| sleep 2 | |
| # Clear WAL and snapshots so ghost Running jobs from previous | |
| # runs don't pollute the new cluster. | |
| rm -rf ~/spur/state/* | |
| ssh mi300-2 'rm -rf ~/spur/state/*' 2>/dev/null || true | |
| - name: Deploy fresh binaries and test scripts | |
| run: | | |
| cp target/release/spur target/release/spurctld target/release/spurd \ | |
| target/release/spurdbd target/release/spurrestd \ | |
| target/release/spur-k8s-operator ~/spur/bin/ | |
| rsync -a --checksum \ | |
| target/release/spur \ | |
| target/release/spurd \ | |
| mi300-2:~/spur/bin/ | |
| # Keep test scripts in sync with the repo on both nodes | |
| chmod +x deploy/bare-metal/inference_job.sh deploy/bare-metal/distributed_job.sh | |
| rsync -a --checksum \ | |
| deploy/bare-metal/inference_test.py \ | |
| deploy/bare-metal/inference_job.sh \ | |
| deploy/bare-metal/distributed_test.py \ | |
| deploy/bare-metal/distributed_job.sh \ | |
| ~/spur/ | |
| rsync -a --checksum \ | |
| deploy/bare-metal/inference_test.py \ | |
| deploy/bare-metal/inference_job.sh \ | |
| deploy/bare-metal/distributed_test.py \ | |
| deploy/bare-metal/distributed_job.sh \ | |
| mi300-2:~/spur/ | |
| - name: Start cluster | |
| run: | | |
| ~/spur/etc/start-controller.sh | |
| ssh mi300-2 '~/spur/etc/start-agent.sh' | |
| sleep 5 | |
| - name: Verify both nodes idle | |
| run: | | |
| ~/spur/bin/sinfo | |
| if ! ~/spur/bin/sinfo | grep -qE "2\s+idle"; then | |
| echo "ERROR: Expected 2 idle nodes, got:" | |
| ~/spur/bin/sinfo | |
| exit 1 | |
| fi | |
| - name: Run integration tests | |
| run: bash deploy/bare-metal/cluster_test.sh | |
| - name: Stop cluster | |
| if: always() | |
| run: | | |
| ssh mi300-2 '~/spur/etc/stop-agent.sh' || true | |
| ~/spur/etc/stop-all.sh || true | |
| build-docker-image: | |
| name: Build Docker Image | |
| runs-on: ubuntu-latest | |
| needs: [build-and-test, fmt, clippy] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build container image | |
| run: sg docker -c "docker build -f deploy/Dockerfile -t spur:ci ." | |
| - name: Save image as artifact | |
| run: sg docker -c "docker save spur:ci -o /tmp/spur-ci-image.tar" | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: spur-ci-image | |
| path: /tmp/spur-ci-image.tar | |
| retention-days: 1 | |
| k8s: | |
| name: K8s Integration | |
| runs-on: [self-hosted, mi300x] | |
| needs: build-docker-image | |
| concurrency: | |
| group: k8s-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download CI image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: spur-ci-image | |
| path: /tmp | |
| - name: Load CI image | |
| run: sg docker -c "docker load -i /tmp/spur-ci-image.tar" | |
| - name: Run K8s integration tests | |
| run: sg docker -c "SPUR_CI_IMAGE=spur:ci bash deploy/bare-metal/k8s_test.sh" | |
| - name: Cleanup kind cluster | |
| if: always() | |
| run: | | |
| export PATH="$HOME/.local/bin:$PATH" | |
| sg docker -c "kind delete cluster --name spur-ci" 2>/dev/null || true |