Skip to content

Commit f17e8be

Browse files
kit1980kit1980
authored
Add reusable workflow for Conda Windows (torchtext only for now) (#695)
1 parent c267d3a commit f17e8be

File tree

2 files changed

+208
-0
lines changed

2 files changed

+208
-0
lines changed

‎.github/workflows/build_conda_windows.yml‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
name: Build Windows Conda
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runner-type:
7+
description: "Runner type"
8+
default: ""
9+
type: string
10+
conda-package-directory:
11+
description: 'Directory where your meta.yaml for your conda package lives'
12+
required: true
13+
type: string
14+
repository:
15+
description: 'Repository to checkout, defaults to ""'
16+
default: ""
17+
type: string
18+
ref:
19+
description: 'Reference to checkout, defaults to "nightly"'
20+
default: "nightly"
21+
type: string
22+
test-infra-repository:
23+
description: "Test infra repository to use"
24+
default: "pytorch/test-infra"
25+
type: string
26+
test-infra-ref:
27+
description: "Test infra reference to use"
28+
default: ""
29+
type: string
30+
build-matrix:
31+
description: "Build matrix to utilize"
32+
default: ""
33+
type: string
34+
pre-script:
35+
description: "Pre script to run prior to build"
36+
default: ""
37+
type: string
38+
post-script:
39+
description: "Post script to run prior to build"
40+
default: ""
41+
type: string
42+
package-name:
43+
description: "Name of the actual python package that is imported"
44+
default: ""
45+
type: string
46+
trigger-event:
47+
description: "Trigger Event in caller that determines whether or not to upload"
48+
default: ""
49+
type: string
50+
51+
jobs:
52+
build:
53+
strategy:
54+
fail-fast: false
55+
matrix: ${{ fromJSON(inputs.build-matrix) }}
56+
env:
57+
CONDA_PACKAGE_DIRECTORY: ${{ inputs.conda-package-directory }}
58+
PYTHON_VERSION: ${{ matrix.python_version }}
59+
PACKAGE_TYPE: conda
60+
REPOSITORY: ${{ inputs.repository }}
61+
REF: ${{ inputs.ref }}
62+
CU_VERSION: ${{ matrix.desired_cuda }}
63+
name: ${{ matrix.build_name }}
64+
runs-on: ${{ inputs.runner-type }}
65+
defaults:
66+
run:
67+
shell: bash -l {0}
68+
# If a build is taking longer than 60 minutes on these runners we need
69+
# to have a conversation
70+
timeout-minutes: 60
71+
steps:
72+
- uses: actions/checkout@v3
73+
with:
74+
# Support the use case where we need to checkout someone's fork
75+
repository: ${{ inputs.test-infra-repository }}
76+
ref: ${{ inputs.test-infra-ref }}
77+
path: test-infra
78+
- uses: ./test-infra/.github/actions/setup-binary-builds
79+
with:
80+
repository: ${{ inputs.repository }}
81+
ref: ${{ inputs.ref }}
82+
setup-miniconda: true
83+
python-version: ${{ env.PYTHON_VERSION }}
84+
- name: Run pre-script
85+
working-directory: ${{ inputs.repository }}
86+
env:
87+
PRE_SCRIPT: ${{ inputs.pre-script }}
88+
if: ${{ inputs.pre-script != '' }}
89+
run: |
90+
if [[ ! -f ${PRE_SCRIPT} ]]; then
91+
echo "::error::Specified pre-script file (${PRE_SCRIPT}) not found, not going execute it"
92+
exit 1
93+
else
94+
${CONDA_RUN} bash "${PRE_SCRIPT}"
95+
fi
96+
- name: Setup base environment variables
97+
run: |
98+
# //\\// means replace every backslash with slash
99+
echo "SOURCE_ROOT_DIR=${GITHUB_WORKSPACE//\\//}/${REPOSITORY}" >> "${GITHUB_ENV}"
100+
- name: VS2019 conda-build
101+
working-directory: ${{ inputs.repository }}
102+
env:
103+
CUDATOOLKIT_CHANNEL: ${{ env.CUDATOOLKIT_CHANNEL }}
104+
PACKAGE_NAME: ${{ inputs.package-name }}
105+
run: |
106+
export VSTOOLCHAIN_PACKAGE=vs2019
107+
export VSDEVCMD_ARGS=''
108+
source "${BUILD_ENV_FILE}"
109+
${CONDA_RUN} conda build \
110+
-c defaults \
111+
-c "${CUDATOOLKIT_CHANNEL}" \
112+
-c "pytorch-${CHANNEL}" \
113+
--no-anaconda-upload \
114+
--python "${PYTHON_VERSION}" \
115+
packaging/$VSTOOLCHAIN_PACKAGE
116+
cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/$PACKAGE_NAME/conda_build_config.yaml
117+
118+
- name: Build the conda (conda-build)
119+
working-directory: ${{ inputs.repository }}
120+
env:
121+
CUDATOOLKIT_CHANNEL: ${{ env.CUDATOOLKIT_CHANNEL }}
122+
run: |
123+
cat "${BUILD_ENV_FILE}"
124+
source "${BUILD_ENV_FILE}"
125+
${CONDA_RUN} conda build \
126+
-c defaults \
127+
-c "${CUDATOOLKIT_CHANNEL}" \
128+
-c "pytorch-${CHANNEL}" \
129+
--no-anaconda-upload \
130+
--python "${PYTHON_VERSION}" \
131+
--output-folder dist/ \
132+
"${CONDA_PACKAGE_DIRECTORY}"
133+
- name: Upload artifact to GitHub
134+
uses: actions/upload-artifact@v3
135+
with:
136+
name: ${{ env.ARTIFACT_NAME }}
137+
path: ${{ inputs.repository }}/dist/
138+
- name: Run post-script
139+
working-directory: ${{ inputs.repository }}
140+
env:
141+
POST_SCRIPT: ${{ inputs.post-script }}
142+
if: ${{ inputs.post-script != '' }}
143+
run: |
144+
if [[ ! -f ${POST_SCRIPT} ]]; then
145+
echo "::error::Specified post-script file (${POST_SCRIPT}) not found, not going execute it"
146+
exit 1
147+
else
148+
${CONDA_RUN} bash "${POST_SCRIPT}"
149+
fi
150+
# TODO: Figure out upload method to s3
151+
152+
concurrency:
153+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}
154+
cancel-in-progress: true

‎.github/workflows/test_build_conda_windows.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: Test Build Windows Conda
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- .github/actions/setup-binary-builds/action.yml
7+
- .github/workflows/test_build_conda_windows.yml
8+
- .github/workflows/build_conda_windows.yml
9+
- .github/workflows/generate_binary_build_matrix.yml
10+
workflow_dispatch:
11+
12+
jobs:
13+
generate-matrix:
14+
uses: ./.github/workflows/generate_binary_build_matrix.yml
15+
with:
16+
package-type: conda
17+
os: windows
18+
test-infra-repository: ${{ github.repository }}
19+
test-infra-ref: ${{ github.ref }}
20+
test:
21+
needs: generate-matrix
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
include:
26+
# Skip torchaudio and torchvision for now.
27+
# - repository: pytorch/audio
28+
# pre-script: packaging/pre_build_script_wheel.sh
29+
# post-script: packaging/post_build_script_wheel.sh
30+
# conda-package-directory: packaging/torchaudio
31+
# - repository: pytorch/vision
32+
# pre-script: ""
33+
# post-script: ""
34+
# package-name: torchvision
35+
# conda-package-directory: packaging/torchvision
36+
- repository: pytorch/text
37+
pre-script: ""
38+
post-script: ""
39+
package-name: torchtext
40+
conda-package-directory: packaging/torchtext
41+
name: ${{ matrix.repository }}
42+
uses: ./.github/workflows/build_conda_windows.yml
43+
with:
44+
runner-type: windows-2019
45+
conda-package-directory: ${{ matrix.conda-package-directory }}
46+
repository: ${{ matrix.repository }}
47+
ref: nightly
48+
test-infra-repository: ${{ github.repository }}
49+
test-infra-ref: ${{ github.ref }}
50+
build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
51+
pre-script: ${{ matrix.pre-script }}
52+
post-script: ${{ matrix.post-script }}
53+
package-name: ${{ matrix.package-name }}
54+
trigger-event: "${{ github.event_name }}"

0 commit comments

Comments
 (0)