Skip to content

Commit d84d4d5

Browse files
samanklesariasamanklesaria
committed
Add torchcodec inspired linux gpu CI workflow
1 parent 0639492 commit d84d4d5

File tree

1 file changed

+182
-93
lines changed

1 file changed

+182
-93
lines changed

‎.github/workflows/unittest-linux-gpu.yml‎

Lines changed: 182 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,98 +1,187 @@
11
name: Unit-tests on Linux GPU
22

33
on:
4-
pull_request:
5-
push:
6-
branches:
7-
- nightly
8-
- main
9-
- release/*
10-
workflow_dispatch:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
workflow_dispatch:
119

1210
jobs:
13-
tests:
14-
strategy:
15-
matrix:
16-
# TODO add up to 3.13
17-
python_version: ["3.10"]
18-
cuda_arch_version: ["12.6"]
19-
fail-fast: false
20-
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
21-
permissions:
22-
id-token: write
23-
contents: read
24-
with:
25-
runner: linux.g5.4xlarge.nvidia.gpu
26-
repository: pytorch/audio
27-
gpu-arch-type: cuda
28-
gpu-arch-version: ${{ matrix.cuda_arch_version }}
29-
timeout: 120
30-
31-
script: |
32-
set -ex
33-
# Set up Environment Variables
34-
export PYTHON_VERSION="${{ matrix.python_version }}"
35-
export PIP_PROGRESS_BAR=off
36-
export CONDA_QUIET=1
37-
export CU_VERSION="${{ matrix.cuda_arch_version }}"
38-
export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
39-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
40-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
41-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
42-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
43-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
44-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true
45-
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true
46-
# Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility
47-
export CUBLAS_WORKSPACE_CONFIG=:4096:8
48-
49-
# Set UPLOAD_CHANNEL
50-
if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
51-
export UPLOAD_CHANNEL=test
52-
else
53-
export UPLOAD_CHANNEL=nightly
54-
fi
55-
56-
echo "::group::Create conda env"
57-
# Mark Build Directory Safe
58-
git config --global --add safe.directory /__w/audio/audio
59-
conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
60-
conda activate ./ci_env
61-
62-
echo "::endgroup::"
63-
echo "::group::Install PyTorch"
64-
export GPU_ARCH_ID="cu126" # TODO this is currently hardcoded, should depend on matrix's value.
65-
PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${GPU_ARCH_ID}"
66-
pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
67-
echo "::endgroup::"
68-
69-
echo "::group::Install TorchAudio"
70-
conda install --quiet --yes cmake ninja
71-
pip3 install --progress-bar off -v -e . --no-use-pep517
72-
73-
echo "::endgroup::"
74-
echo "::group::Build FFmpeg"
75-
.github/scripts/ffmpeg/build_gpu.sh
76-
77-
echo "::endgroup::"
78-
echo "::group::Install other Dependencies"
79-
80-
pip3 install parameterized requests coverage pytest pytest-cov scipy numpy expecttest
81-
82-
echo "::endgroup::"
83-
echo "::group::Run tests"
84-
export PATH="${PWD}/third_party/install/bin/:${PATH}"
85-
86-
declare -a args=(
87-
'-v'
88-
'--cov=torchaudio'
89-
"--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml"
90-
'--durations' '100'
91-
'-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency'
92-
)
93-
94-
cd test
95-
python3 -m torch.utils.collect_env
96-
env | grep TORCHAUDIO || true
97-
pytest "${args[@]}" torchaudio_unittest
98-
coverage html
11+
build:
12+
# Do not use matrix here to parameterize Python/CUDA versions.
13+
# This job is required to pass for each PR.
14+
# The name of the required job is sensitive to matrix parameter.
15+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
16+
permissions:
17+
id-token: write
18+
contents: read
19+
with:
20+
job-name: Build doc
21+
runner: linux.g5.4xlarge.nvidia.gpu
22+
repository: pytorch/audio
23+
gpu-arch-type: cuda
24+
gpu-arch-version: "12.6" # See GPU_ARCH_ID below
25+
timeout: 120
26+
27+
script: |
28+
set -ex
29+
30+
# Set up Environment Variables
31+
export PYTHON_VERSION="3.10"
32+
export CU_VERSION="11.8"
33+
export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
34+
export PIP_PROGRESS_BAR=off
35+
export CONDA_QUIET=1
36+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
37+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
38+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
39+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
40+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
41+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true
42+
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true
43+
# Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility
44+
export CUBLAS_WORKSPACE_CONFIG=:4096:8
45+
46+
# Set CHANNEL
47+
if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
48+
export CHANNEL=test
49+
export BUILD_VERSION="$( cut -f 1 -d a version.txt )"
50+
else
51+
export CHANNEL=nightly
52+
export BUILD_VERSION="$( cut -f 1 -d a version.txt )".dev"$(date "+%Y%m%d")"
53+
fi
54+
55+
# Install miniforge
56+
wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
57+
bash Miniforge3.sh -b -p "${HOME}/conda"
58+
source "${HOME}/conda/etc/profile.d/conda.sh"
59+
60+
echo "::group::Create conda env"
61+
# Mark Build Directory Safe
62+
git config --global --add safe.directory /__w/audio/audio
63+
conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
64+
conda activate ./ci_env
65+
conda install -q -y pip numpy
66+
67+
echo "::endgroup::"
68+
echo "::group::Install PyTorch"
69+
70+
GPU_ARCH_ID=cu126 # This is hard-coded and must be consistent with gpu-arch-version.
71+
PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
72+
pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
73+
74+
echo "::endgroup::"
75+
echo "::group::Install TorchAudio"
76+
conda install --quiet --yes cmake>=3.18.0 ninja
77+
pip install --progress-bar off -v . --no-build-isolation
78+
79+
# TODO: Need to rely on torchcodec instead of building ffmpeg from source.
80+
echo "::endgroup::"
81+
echo "::group::Build FFmpeg"
82+
conda install -q -y "ffmpeg<=7"
83+
84+
echo "::endgroup::"
85+
echo "::group::Install other dependencies"
86+
87+
pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest
88+
89+
declare -a args=(
90+
'-v'
91+
'--cov=torchaudio'
92+
"--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml"
93+
'--durations' '100'
94+
'-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency'
95+
)
96+
97+
cd test
98+
python3 -m torch.utils.collect_env
99+
env | grep TORCHAUDIO || true
100+
pytest "${args[@]}" torchaudio_unittest
101+
coverage html
102+
103+
commit-main:
104+
if: github.ref_name == 'main'
105+
permissions:
106+
# Required for `git push`
107+
# Note:
108+
# This is not effective from fork.
109+
# When you debug this, make sure to make a branch on pytorch and
110+
# make PR from there.
111+
contents: write
112+
runs-on: ubuntu-latest
113+
needs: build
114+
steps:
115+
- uses: actions/checkout@v3
116+
with:
117+
ref: gh-pages
118+
fetch-depth: 5
119+
- uses: actions/download-artifact@v3
120+
with:
121+
name: docs
122+
- name: Update main doc
123+
run: |
124+
set -x
125+
126+
git config user.name "pytorchbot"
127+
git config user.email "[email protected]"
128+
# When `git clone`, `gh-pages` branch is fetched by default.
129+
# The size of gh-pages grows significantly, so we use ammend and force push
130+
# We add a new commit once a week
131+
if [ "$(date +%d)" = "1" ]; then
132+
git commit --allow-empty -m "placeholder"
133+
fi
134+
135+
# TODO: add tag-based process (need to handle the main directory name)
136+
# Update the main doc
137+
rm -rf main
138+
mv html main
139+
git add --all main || true
140+
141+
git commit --amend -m "auto-generating sphinx docs" || true
142+
git push -f
143+
144+
# Push for release
145+
# Make sure that version.txt is updated first (alpha suffix is removed)
146+
commit-release:
147+
if: startsWith(github.ref_name, 'release/') && ( github.event_name == 'workflow_dispatch' )
148+
permissions:
149+
# Required for `git push`
150+
# Note:
151+
# This is not effective from fork.
152+
# When you debug this, make sure to make a branch on pytorch and
153+
# make PR from there.
154+
contents: write
155+
runs-on: ubuntu-latest
156+
needs: build
157+
steps:
158+
- uses: actions/checkout@v3
159+
with:
160+
ref: gh-pages
161+
fetch-depth: 5
162+
- uses: actions/checkout@v4
163+
with:
164+
path: _src
165+
- uses: actions/download-artifact@v3
166+
with:
167+
name: docs
168+
- name: Update doc
169+
run: |
170+
set -x
171+
172+
git config user.name "pytorchbot"
173+
git config user.email "[email protected]"
174+
# When `git clone`, `gh-pages` branch is fetched by default.
175+
# The size of gh-pages grows significantly, so we use ammend and force push
176+
# We add a new commit once a week
177+
if [ "$(date +%d)" = "1" ]; then
178+
git commit --allow-empty -m "placeholder"
179+
fi
180+
181+
dirname="$(cat _src/version.txt)"
182+
rm -rf "${dirname}"
183+
mv html "${dirname}"
184+
git add --all "${dirname}" || true
185+
186+
git commit --amend -m "auto-generating sphinx docs" || true
187+
git push -f

0 commit comments

Comments
 (0)