Build all RAPIDS repositories #776
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build all RAPIDS repositories | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_call: | |
| jobs: | |
| check-event: | |
| name: Check GH Event | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ok: ${{ steps.check_gh_event.outputs.ok }} | |
| steps: | |
| - id: check_gh_event | |
| name: Check GH Event | |
| shell: bash | |
| run: | | |
| [[ '${{ github.event_name }}' == 'workflow_dispatch' ]] || \ | |
| [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ | |
| [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ | |
| [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'rapidsai/devcontainers' ]] \ | |
| && echo "ok=true" | tee -a "$GITHUB_OUTPUT" \ | |
| || echo "ok=false" | tee -a "$GITHUB_OUTPUT"; | |
| build-all-rapids-repos: | |
| name: ${{ matrix.libs }} | |
| if: needs.check-event.outputs.ok == 'true' | |
| needs: check-event | |
| secrets: inherit | |
| uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@main | |
| permissions: | |
| actions: read | |
| packages: read | |
| id-token: write | |
| contents: read | |
| pull-requests: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - libs: "rmm ucxx kvikio dask-cuda cudf cudf_kafka rapidsmpf" | |
| - libs: "rmm ucxx dask-cuda raft cuvs cuml nvforest cuopt" | |
| - libs: "rmm ucxx dask-cuda raft cugraph cugraph-gnn nx-cugraph" | |
| with: | |
| arch: '["amd64", "arm64"]' | |
| cuda: '["12.9", "13.1"]' | |
| node_type: cpu16 | |
| timeout-minutes: 720 | |
| # 1. Prohibit sccache from shutting down automatically | |
| # 2. Infinitely retry transient errors | |
| # 3. Enable debug logging to track cache misses | |
| # 4. Never fallback to locally compiling | |
| env: | | |
| PYTHON_VERSION=3.13 | |
| CONDA_ENV_CREATE_QUIET=1 | |
| INCLUDE_REPOS="${{ matrix.libs }}" | |
| PARALLEL_LEVEL=0 | |
| SCCACHE_IDLE_TIMEOUT=0 | |
| SCCACHE_SERVER_LOG=sccache=debug | |
| SCCACHE_DIST_MAX_RETRIES=inf | |
| SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false | |
| build_command: | | |
| function begin_group() { | |
| local blue="34" | |
| echo -e "::group::\e[${blue}m${1:-}\e[0m" | |
| } | |
| function end_group() { | |
| local name="${1:-}" | |
| local build_status="${2:-0}" | |
| local red="31" | |
| echo "::endgroup::" | |
| if [ "$build_status" -ne 0 ]; then | |
| echo -e "::error::\e[${red}m ${name} - Failed (⬆️ click above for full log ⬆️)\e[0m" | |
| fi | |
| } | |
| function run_command() { | |
| local -; | |
| set -euo pipefail; | |
| local group="${1:-}"; | |
| shift; | |
| local command=("$@"); | |
| local exit_code="0"; | |
| begin_group "$group"; | |
| echo "Working directory: $(pwd)"; | |
| echo "Running command: ${command[*]}"; | |
| "${command[@]}" || exit_code=$?; | |
| end_group "$group" "$exit_code" | |
| return "$exit_code" | |
| } | |
| # Clone all the repos | |
| run_command "Clone RAPIDS repositories" \ | |
| clone-all -j$(nproc) -v -q --clone-upstream --depth 1 --single-branch --shallow-submodules --no-update-env; | |
| run_command "Create RAPIDS python environment" \ | |
| rapids-post-start-command; | |
| # Configure all the C++ libs | |
| run_command "Configure C++ libraries" bash -ceuo pipefail "\ | |
| time configure-all \ | |
| -j${PARALLEL_LEVEL} \ | |
| -GNinja \ | |
| -Wno-dev \ | |
| -DBUILD_TESTS=ON \ | |
| -DBUILD_BENCHMARKS=ON \ | |
| -DBUILD_PRIMS_BENCH=ON \ | |
| -DBUILD_SHARED_LIBS=ON \ | |
| -DRAFT_COMPILE_LIBRARY=ON \ | |
| -DBUILD_CUGRAPH_MG_TESTS=ON 2>&1 \ | |
| | tee -a telemetry-artifacts/build.log"; | |
| # Build all the C++ libs | |
| run_command "Build C++ libraries" bash -ceuo pipefail "\ | |
| time build-all-cpp -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; | |
| # Build all the Python libs | |
| run_command "Build Python libraries" bash -ceuo pipefail "\ | |
| time build-all-python -j${PARALLEL_LEVEL} 2>&1 | tee -a telemetry-artifacts/build.log"; | |
| # Print cache and dist stats | |
| run_command "sccache stats" bash -ceuo pipefail "\ | |
| sccache --show-adv-stats | tee -a telemetry-artifacts/sccache-stats.txt"; | |
| # Print build times | |
| run_command "Build times" bash -ceuo pipefail "\ | |
| find /var/log/devcontainer-utils/ -type f -name 'build-*-time.log' -print0 \ | |
| | xargs -0 -n1 grep -H real | sed 's/real\t/ /g' || :" # Nonfatal if not found |