perf: setup codspeed to track performance regression (#4519)

Co-authored-by: remimimimimi <[email protected]>
Co-authored-by: Tim de Jager <[email protected]>
Co-authored-by: Felix Häcker <[email protected]>
Co-authored-by: Hofer-Julian <[email protected]>
Co-authored-by: Lucas Colley <[email protected]>
Co-authored-by: Bas Zalmstra <[email protected]>
Co-authored-by: Julian Hofer <[email protected]>
Co-authored-by: Valentin Kharin <[email protected]>

Author: 7 people

.github/actionlint.yml

@@ -1,5 +1,6 @@
 self-hosted-runner:
   labels:
+    - "codspeed-macro"
     - 8core_ubuntu_latest_runner
     - 16core_windows_latest_runner
     - windows_arm64_2025_large

.github/workflows/benchmark.yml

@@ -0,0 +1,43 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - "feature/codespeed" # or "master"
+  pull_request: # required to have reports on PRs
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+    env:
+      RUST_BACKTRACE: full
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tar bzip2
+
+      - name: Setup rust toolchain, cache and cargo-codspeed binary
+        uses: moonrepo/setup-rust@e013866c4215f77c925f42f60257dec7dd18836e
+        with:
+          channel: stable
+          cache-target: release
+          bins: cargo-codspeed
+
+      - name: Build the benchmark target(s)
+        run: cargo codspeed build -p pixi_bench
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@cc824aeb2c86848c39cf722ab4c2b6c5bf290530
+        with:
+          run: |
+            export PATH="~/.cargo/bin:$PATH"
+            cargo codspeed run -p pixi_bench
+          mode: walltime
+          token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/ci.yml

@@ -107,7 +107,7 @@ jobs:
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
       - run: |
-          for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | .name'); do
+          for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name != "pixi_bench") | .name'); do
             cargo rustdoc -p "$package" --all-features -- -D warnings -W unreachable-pub
           done