[lora] refactor - native LoRA #8292
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # doc-dev: docs/ci/00-stage.md | |
| # doc-dev: docs/ci/01-label.md | |
| name: PR Test | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review, labeled] | |
| schedule: | |
| # Nightly CI on main runs every enabled tag except ft-long, with fast-fail disabled. | |
| # 15:00 UTC = 08:00 PDT / 07:00 PST (cron is UTC, no DST). | |
| - cron: '0 15 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| infinite_run: | |
| description: 'Run training infinitely' | |
| required: false | |
| type: boolean | |
| default: false | |
| ci_megatron_pr: | |
| description: 'Megatron-LM branch/commit (default: miles-main)' | |
| required: false | |
| type: string | |
| default: 'miles-main' | |
| ci_sglang_pr: | |
| description: 'SGLang branch/commit (default: sglang-miles)' | |
| required: false | |
| type: string | |
| default: 'sglang-miles' | |
| ci_image_tag: | |
| description: 'Miles Docker image tag (default: dev)' | |
| required: false | |
| type: string | |
| default: 'dev' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # PR updates supersede their previous run; distinct cron schedules and | |
| # manual operations must not cancel one another on the default branch. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.schedule || github.run_id }} | |
| cancel-in-progress: true | |
| # `resolve-ci-policy` passes trigger-specific facts to `ci_policy.py`, which | |
| # owns the explicit cadence, raw labels, test scope, and fast-fail policy. | |
| # `run_suite.py` consumes that shared policy; GPU jobs consume the shared | |
| # bypass output for their cross-stage gate. | |
| # | |
| # Trigger type is not policy: the current nightly cron is mapped by its exact | |
| # expression, while workflow_dispatch remains a regular manual operation with | |
| # no implicit domain scope. | |
| jobs: | |
| resolve-ci-policy: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cadence: ${{ steps.resolve.outputs.cadence }} | |
| raw_labels: ${{ steps.resolve.outputs.raw_labels }} | |
| bypass_fastfail: ${{ steps.resolve.outputs.bypass_fastfail }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Resolve CI policy inputs | |
| id: resolve | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| SCHEDULE: ${{ github.event.schedule || '' }} | |
| PR_LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }} | |
| run: python -m tests.ci.ci_policy | |
| docker-paths: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| changed: ${{ steps.diff.outputs.changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - id: diff | |
| shell: bash | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} | |
| run: | | |
| # Cron and manual runs have no PR diff and never build a PR image. | |
| if [ -z "$BASE_SHA" ]; then | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Compare the PR merge commit with its current base (HEAD^1) rather than | |
| # the recorded base.sha, which can lag main and pull in unrelated changes. | |
| # Dockerfile.rocm doesn't feed the cu13 multi-arch build this pipeline produces. | |
| if git diff --name-only HEAD^1 HEAD \ | |
| | grep -qE '^(docker/(Dockerfile|build\.py|verify_transformer_engine\.py)|requirements\.txt)$|^docker/patch/'; then | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Docker PRs: build the multi-arch image first and run every suite inside it. | |
| # Always runs: with nothing to build every step is a no-op and it stays off the | |
| # GPU fleet. | |
| docker-build: | |
| needs: [docker-paths] | |
| if: always() && !cancelled() | |
| runs-on: ${{ fromJSON((needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository) && '["h200", "2gpu"]' || '["ubuntu-latest"]') }} | |
| timeout-minutes: 180 | |
| outputs: | |
| built: ${{ steps.build.outputs.built || 'false' }} | |
| env: | |
| # Forks cannot push to the registry, so they keep the released image. | |
| BUILD: ${{ needs.docker-paths.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository }} | |
| steps: | |
| - name: Checkout repository | |
| if: env.BUILD == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| if: env.BUILD == 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| network=host | |
| - name: Install Python + dependencies | |
| if: env.BUILD == 'true' | |
| run: | | |
| apt-get update && apt-get install -y python3 python3-pip | |
| pip3 install --break-system-packages typer | |
| - name: Login to Docker Hub | |
| if: env.BUILD == 'true' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push PR tag | |
| id: build | |
| if: env.BUILD == 'true' | |
| run: | | |
| python3 docker/build.py --variant cu13 --image-tag custom \ | |
| --custom-tag pr-${{ github.event.pull_request.number }} --push | |
| echo "built=true" >> "$GITHUB_OUTPUT" | |
| - name: Nothing to build | |
| if: env.BUILD != 'true' | |
| run: echo "No docker-relevant changes; suites use the released image." | |
| # No `if:` guard: a real build failure must stop the suites rather than run them | |
| # against a stale image. | |
| resolve-ci-image: | |
| needs: [docker-build] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| container_image: ${{ steps.resolve.outputs.container_image }} | |
| steps: | |
| - name: Resolve CI image | |
| id: resolve | |
| shell: bash | |
| env: | |
| PR_BODY: ${{ github.event.pull_request.body || '' }} | |
| INPUT_CI_IMAGE_TAG: ${{ github.event.inputs.ci_image_tag || '' }} | |
| DOCKER_BUILT: ${{ needs.docker-build.outputs.built }} | |
| PR_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| run: | | |
| CI_IMAGE_TAG="${INPUT_CI_IMAGE_TAG}" | |
| [ -z "$CI_IMAGE_TAG" ] && [ "$DOCKER_BUILT" = "true" ] && CI_IMAGE_TAG="pr-${PR_NUMBER}" | |
| if [ -n "$PR_BODY" ]; then | |
| PR_CI_IMAGE_TAG=$(echo "$PR_BODY" | grep -m1 -oP '^ci-image-tag:\s+\K\S+' || true) | |
| [ -z "$CI_IMAGE_TAG" ] && [ -n "$PR_CI_IMAGE_TAG" ] && CI_IMAGE_TAG="$PR_CI_IMAGE_TAG" | |
| fi | |
| [ -z "$CI_IMAGE_TAG" ] && CI_IMAGE_TAG="dev" | |
| if [[ ! "$CI_IMAGE_TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then | |
| echo "::error::ci-image-tag must be a Docker tag, not a full image name: $CI_IMAGE_TAG" | |
| exit 1 | |
| fi | |
| echo "container_image=radixark/miles:${CI_IMAGE_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "Resolved CI image: radixark/miles:${CI_IMAGE_TAG}" | |
| # Stage A: CPU-only fast tests (always runs on PR) | |
| # Runs on GitHub-hosted ubuntu-latest (cpu_runner: true) to avoid | |
| # occupying GPU-fleet runner slots. | |
| stage-a-cpu: | |
| needs: [resolve-ci-policy, resolve-ci-image] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition_id: [0, 1, 2, 3] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| cpu_runner: true | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cpu --suite stage-a-cpu | |
| --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 4 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| # Stage B: CPU-only bucket for slower CPU tests that don't fit stage-a-cpu's | |
| # fast budget. Always runs on PR; an empty suite still exits 0 in run_suite.py. | |
| stage-b-cpu: | |
| needs: [resolve-ci-policy, resolve-ci-image] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| cpu_runner: true | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cpu --suite stage-b-cpu | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| stage-b-2-gpu-h200: | |
| needs: [resolve-ci-policy, resolve-ci-image, stage-a-cpu] | |
| if: | | |
| always() && !cancelled() && | |
| needs.resolve-ci-policy.result == 'success' && | |
| needs.resolve-ci-image.result == 'success' && | |
| (needs.stage-a-cpu.result == 'success' || | |
| (needs.stage-a-cpu.result == 'failure' && needs.resolve-ci-policy.outputs.bypass_fastfail == 'true')) | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| runs_on: '["h200", "2gpu"]' | |
| container_image: ${{ needs.resolve-ci-image.outputs.container_image }} | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cuda --suite stage-b-2-gpu-h200 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| stage-c-8-gpu-h100: | |
| needs: [resolve-ci-policy, resolve-ci-image, stage-a-cpu] | |
| if: | | |
| always() && !cancelled() && | |
| needs.resolve-ci-policy.result == 'success' && | |
| needs.resolve-ci-image.result == 'success' && | |
| (needs.stage-a-cpu.result == 'success' || | |
| (needs.stage-a-cpu.result == 'failure' && needs.resolve-ci-policy.outputs.bypass_fastfail == 'true')) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition_id: [0, 1] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| runs_on: '["h100", "8gpu"]' | |
| container_image: ${{ needs.resolve-ci-image.outputs.container_image }} | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cuda --suite stage-c-8-gpu-h100 | |
| --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| stage-c-8-gpu-h200: | |
| needs: [resolve-ci-policy, resolve-ci-image, stage-a-cpu] | |
| if: | | |
| always() && !cancelled() && | |
| needs.resolve-ci-policy.result == 'success' && | |
| needs.resolve-ci-image.result == 'success' && | |
| (needs.stage-a-cpu.result == 'success' || | |
| (needs.stage-a-cpu.result == 'failure' && needs.resolve-ci-policy.outputs.bypass_fastfail == 'true')) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition_id: [0, 1] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| runs_on: '["h200", "8gpu"]' | |
| container_image: ${{ needs.resolve-ci-image.outputs.container_image }} | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cuda --suite stage-c-8-gpu-h200 | |
| --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| stage-c-4-gpu-h200: | |
| needs: [resolve-ci-policy, resolve-ci-image, stage-a-cpu] | |
| if: | | |
| always() && !cancelled() && | |
| needs.resolve-ci-policy.result == 'success' && | |
| needs.resolve-ci-image.result == 'success' && | |
| (needs.stage-a-cpu.result == 'success' || | |
| (needs.stage-a-cpu.result == 'failure' && needs.resolve-ci-policy.outputs.bypass_fastfail == 'true')) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition_id: [0, 1, 2] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| runs_on: '["h200", "4gpu"]' | |
| container_image: ${{ needs.resolve-ci-image.outputs.container_image }} | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cuda --suite stage-c-4-gpu-h200 | |
| --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 3 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit | |
| stage-c-2-gpu-h200: | |
| needs: [resolve-ci-policy, resolve-ci-image, stage-a-cpu] | |
| if: | | |
| always() && !cancelled() && | |
| needs.resolve-ci-policy.result == 'success' && | |
| needs.resolve-ci-image.result == 'success' && | |
| (needs.stage-a-cpu.result == 'success' || | |
| (needs.stage-a-cpu.result == 'failure' && needs.resolve-ci-policy.outputs.bypass_fastfail == 'true')) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition_id: [0, 1] | |
| uses: ./.github/workflows/_run-ci.yml | |
| with: | |
| runs_on: '["h200", "2gpu"]' | |
| container_image: ${{ needs.resolve-ci-image.outputs.container_image }} | |
| execute_command: >- | |
| python tests/ci/run_suite.py --hw cuda --suite stage-c-2-gpu-h200 | |
| --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 | |
| --cadence ${{ needs.resolve-ci-policy.outputs.cadence }} | |
| --labels ${{ needs.resolve-ci-policy.outputs.raw_labels }} | |
| secrets: inherit |