Integration Tests #241
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: "30 6 * * 1" # every Monday at 6:30 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: "PR number to comment metrics on" | |
| type: number | |
| required: false | |
| methods: | |
| description: 'Optional test filer (e.g. "edgebank", "linkprop")' | |
| type: string | |
| required: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| integration-tests-slurm: | |
| name: Integration testing on a SLURM cluster | |
| runs-on: self-hosted | |
| timeout-minutes: 300 | |
| if: github.actor == 'jacob-chmura' || github.actor == 'shenyanghuang' || github.actor == 'benjaminnNgo' | |
| env: | |
| REAL_HOME: /home/mila/j/jacob.chmura | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set Runtime Environment Variables | |
| run: | | |
| echo "TGB_DATA_ROOT=$REAL_HOME/tgb_datasets" >> $GITHUB_ENV | |
| echo "GRAPH_PROP_PRED_DATA_ROOT=$REAL_HOME/graph_prop_datasets" >> $GITHUB_ENV | |
| echo "TGM_CI_LOG_BASE=$REAL_HOME/tgm_ci" >> $GITHUB_ENV | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: ".python-version" | |
| - name: Set up uv | |
| run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh | |
| - name: Restore uv cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.uv-cache | |
| key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | |
| uv-${{ runner.os }} | |
| - name: Download TGB datasets | |
| id: submit-download | |
| run: | | |
| echo "Preparing TGB datasets in $TGB_DATA_ROOT..." | |
| JOB_ID=$(sbatch --parsable .github/scripts/run_download_tgb_job_sbatch.sh "$TGB_DATA_ROOT") | |
| echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT | |
| - name: Download sample datasets for Graph Property Prediction examples | |
| run: | | |
| echo "Preparing Graph Property prediction datasets in $GRAPH_PROP_PRED_DATA_ROOT ..." | |
| ./scripts/download_graph_prop_datasets.sh "$GRAPH_PROP_PRED_DATA_ROOT" | |
| - name: Wait for Download TGB job to finish | |
| run: | | |
| JOB_ID="${{ steps.submit-download.outputs.job_id }}" | |
| .github/scripts/poll_slurm_job.sh "$JOB_ID" --timeout 3600 --interval 10 | |
| - name: Install dependencies | |
| run: uv sync --group examples | |
| - name: Run Integration Tests | |
| run: | | |
| if [ -n "${{ github.event.inputs.methods }}" ]; then | |
| echo "Running filtered integration tests: ${{ github.event.inputs.methods }}" | |
| uv run pytest test/integration -m "integration" -n 2 -vvv -k "${{ github.event.inputs.methods }}" | |
| else | |
| echo "Running all integration tests" | |
| uv run pytest test/integration -m "integration" -n 2 -vvv | |
| fi | |
| - name: Get Latest CI Run Log Path | |
| if: always() | |
| id: get-log-dir | |
| run: | | |
| # Parse path + ID written by the test runner (2 lines in ~/tgm_ci/latest_path.txt) | |
| mapfile -t lines < "$TGM_CI_LOG_BASE/latest_path.txt" | |
| LOG_PATH="${lines[0]}" | |
| CI_RUN_DIR="${lines[1]}" | |
| echo "log_path=$LOG_PATH" >> "$GITHUB_OUTPUT" | |
| echo "ci_run_dir=$CI_RUN_DIR" >> "$GITHUB_OUTPUT" | |
| echo "Resolved log path: $LOG_PATH" | |
| echo "Resolved run dir: $CI_RUN_DIR" | |
| - name: Parse structured logs | |
| run: | | |
| set -euo pipefail | |
| LOG_DIR="${{ steps.get-log-dir.outputs.log_path }}" | |
| for log_file in "$LOG_DIR"/*.log; do | |
| ./scripts/parse_log_to_json.sh "$log_file" | |
| done | |
| - name: Merge structured logs | |
| run: | | |
| LOG_DIR="${{ steps.get-log-dir.outputs.log_path }}" | |
| ./.github/scripts/merge_json_files.py "$LOG_DIR" | |
| - name: Zip raw logs | |
| if: ${{ github.event.inputs.pr_number }} | |
| id: zip-raw-logs | |
| run: | | |
| LOG_DIR="${{ steps.get-log-dir.outputs.log_path }}" | |
| ZIP_FILE="$LOG_DIR/raw_logs.zip" | |
| cd "$LOG_DIR" | |
| zip -r raw_logs.zip *.out *.err *.log | |
| echo "zip_file=$ZIP_FILE" >> $GITHUB_OUTPUT | |
| - name: Upload all logs and parsed JSON | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ci-artifacts | |
| path: | | |
| ${{ steps.get-log-dir.outputs.log_path }}/*.out | |
| ${{ steps.get-log-dir.outputs.log_path }}/*.err | |
| ${{ steps.get-log-dir.outputs.log_path }}/*.log | |
| ${{ steps.get-log-dir.outputs.log_path }}/*.json | |
| ${{ steps.zip-raw-logs.outputs.zip_file }} | |
| - name: Comment metrics + raw logs on PR | |
| if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number }} | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const pr_number = parseInt("${{ github.event.inputs.pr_number }}"); | |
| const logDir = "${{ steps.get-log-dir.outputs.log_path }}"; | |
| const metricsPath = path.join(logDir, "metrics.json"); | |
| const metrics = JSON.parse(fs.readFileSync(metricsPath, "utf8")); | |
| let body = ""; | |
| for (const [method, vals] of Object.entries(metrics)) { | |
| body += `#### Job: ${method}\n\n`; | |
| body += `| Metric | Value |\n|--------|-------|\n`; | |
| for (const [metric, value] of Object.entries(vals)) { | |
| let displayValue = value; | |
| if (typeof value === "number") { | |
| displayValue = value.toFixed(3); | |
| } | |
| body += `| ${metric} | ${displayValue} |\n`; | |
| } | |
| body += "\n"; | |
| } | |
| const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| body += `[Download raw logs](${runUrl})`; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: pr_number, | |
| body: body | |
| }); | |
| - name: Minimize uv cache | |
| run: uv cache prune --ci |