-
Notifications
You must be signed in to change notification settings - Fork 6
284 lines (281 loc) · 10.7 KB
/
transformer-docker.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
name: Transformer Docker Images
on:
workflow_dispatch:
inputs:
build_echo_image:
description: 'Build transformer_echo and transformer_echo_go image'
required: true
type: boolean
default: false
build_hello_world_image:
description: 'Build hello_world image'
required: true
type: boolean
default: false
build_md5_image:
description: 'Build transformer_md5 image'
required: true
type: boolean
default: false
build_tar2tf_image:
description: 'Build transformer_tar2tf image'
required: true
type: boolean
default: false
build_compress_image:
description: 'Build transformer_compress image'
required: true
type: boolean
default: false
build_ffmpeg_image:
description: 'Build transformer_nemo_ffmpeg image'
required: true
type: boolean
default: false
build_split_audio_image:
description: 'Build transformer_split_audio image'
required: true
type: boolean
default: false
build_torchvision_preprocess_image:
description: 'Build transformer_torchvision image'
required: true
type: boolean
default: false
build_keras_preprocess_image:
description: 'Build transformer_keras image'
required: true
type: boolean
default: false
build_face_detection_image:
description: 'Build transformer_face_detection image'
required: true
type: boolean
default: false
push:
paths:
- 'transformers/**'
env:
ECHO_ENABLE: ${{ github.event.inputs.build_echo_image }}
HELLO_WORLD_ENABLE: ${{ github.event.inputs.build_hello_world_image }}
MD5_ENABLE: ${{ github.event.inputs.build_md5_image }}
TAR2TF_ENABLE: ${{ github.event.inputs.build_tar2tf_image }}
COMPRESS_ENABLE: ${{ github.event.inputs.build_compress_image }}
FFMPEG_ENABLE: ${{ github.event.inputs.build_ffmpeg_image }}
TORCHVISION_ENABLE: ${{ github.event.inputs.build_torchvision_preprocess_image }}
KERAS_ENABLE: ${{ github.event.inputs.build_keras_preprocess_image }}
FACE_DETECTION_ENABLE: ${{ github.event.inputs.build_face_detection_image }}
SPLIT_AUDIO_ENABLE: ${{ github.event.inputs.build_split_audio_image }}
jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Docker Login
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup Minikube
uses: medyagh/[email protected]
with:
driver: docker
container-runtime: docker
cpus: max
memory: max
- name: Run AIStore in Minikube
run: |
git clone https://github.com/NVIDIA/aistore.git
cd aistore/deploy/dev/k8s
./minimal.sh
export AIS_ENDPOINT="http://$(minikube ip):8080"
- name: Setup Python 3.11
uses: actions/setup-python@v2
with:
python-version: 3.11
- name: Check Path for Changes
id: paths_filter
if: ${{ github.event_name != 'workflow_dispatch' }}
uses: dorny/paths-filter@v2
with:
filters: |
echo:
- transformers/echo/**/*
- transformers/go_echo/**/*
- transformers/tests/test_echo.py
- transformers/tests/test_go_echo.py
hello_world:
- transformers/hello_world/**/*
- transformers/go_hello_world/**/*
- transformers/tests/test_hello_world.py
md5:
- transformers/md5/**/*
- transformers/tests/test_md5.py
tar2tf:
- transformers/tar2tf/**/*
- transformers/tests/test_tar2tf.py
compress:
- transformers/compress/**/*
- transformers/tests/test_compress.py
ffmpeg:
- transformers/ffmpeg/**/*
- transformers/tests/test_ffmpeg.py
torchvision:
- transformers/torchvision_preprocess/**/*
- transformers/tests/test_torchvision_transformer.py
keras:
- transformers/keras_preprocess/**/*
- transformers/tests/test_keras_transformer.py
face_detection:
- transformers/face_detection/**/*
- transformers/tests/test_face_detection.py
- name: Update var ECHO_ENABLE
if: steps.paths_filter.outputs.echo == 'true'
run: echo "ECHO_ENABLE=true" >> $GITHUB_ENV
- name: Update var HELLO_WORLD_ENABLE
if: steps.paths_filter.outputs.hello_world == 'true'
run: echo "HELLO_WORLD_ENABLE=true" >> $GITHUB_ENV
- name: Update var MD5_ENABLE
if: steps.paths_filter.outputs.md5 == 'true'
run: echo "MD5_ENABLE=true" >> $GITHUB_ENV
- name: Update var TAR2TF_ENABLE
if: steps.paths_filter.outputs.tar2tf == 'true'
run: echo "TAR2TF_ENABLE=true" >> $GITHUB_ENV
- name: Update var COMPRESS_ENABLE
if: steps.paths_filter.outputs.compress == 'true'
run: echo "COMPRESS_ENABLE=true" >> $GITHUB_ENV
- name: Update var FFMPEG_ENABLE
if: steps.paths_filter.outputs.ffmpeg == 'true'
run: echo "FFMPEG_ENABLE=true" >> $GITHUB_ENV
- name: Update var TORCHVISION_ENABLE
if: steps.paths_filter.outputs.torchvision == 'true'
run: echo "TORCHVISION_ENABLE=true" >> $GITHUB_ENV
- name: Update var KERAS_ENABLE
if: steps.paths_filter.outputs.keras == 'true'
run: echo "KERAS_ENABLE=true" >> $GITHUB_ENV
- name: Update var FACE_DETECTION_ENABLE
if: steps.paths_filter.outputs.face_detection == 'true'
run: echo "FACE_DETECTION_ENABLE=true" >> $GITHUB_ENV
- name: Install Python Dependencies for Testing
run: |
make -B -C $GITHUB_WORKSPACE/transformers common_deps
- name: Build and Test ECHO
if: ${{ env.ECHO_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/hello_world all
make -B -C $GITHUB_WORKSPACE/transformers/go_hello_world all
pytest -v test_echo.py test_go_echo.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/echo all
make -B -C $GITHUB_WORKSPACE/transformers/go_echo all
pytest -v test_echo.py test_go_echo.py
popd
- name: Build and Test HELLO_WORLD
if: ${{ env.HELLO_WORLD_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/echo all
make -B -C $GITHUB_WORKSPACE/transformers/go_echo all
pytest -v test_hello_world.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/echo all
make -B -C $GITHUB_WORKSPACE/transformers/go_echo all
pytest -v test_hello_world.py
popd
- name: Build and Test MD5
if: ${{ env.MD5_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/md5 all
pytest -v test_md5.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/md5 all
pytest -v test_md5.py
popd
- name: Build and Test TAR2TF
if: ${{ env.TAR2TF_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all
pytest -v test_tar2tf.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/tar2tf all
pytest -v test_tar2tf.py
popd
- name: Build and Test COMPRESS
if: ${{ env.COMPRESS_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/compress all
pytest -v test_compress.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/compress all
pytest -v test_compress.py
popd
- name: Build and Test FFmpeg
if: ${{ env.FFMPEG_ENABLE == 'true' }}
run: |
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/NeMo/FFmpeg all
- name: Build and Test Split Audio
if: ${{ env.SPLIT_AUDIO_ENABLE == 'true' }}
run: |
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/NeMo/split_audio all
- name: Build and Test TORCHVISION
if: ${{ env.TORCHVISION_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pip install torchvision --index-url https://download.pytorch.org/whl/cpu
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all
pytest -v test_torchvision_transformer.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/torchvision_preprocess all
pytest -v test_torchvision_transformer.py
popd
- name: Build and Test KERAS
if: ${{ env.KERAS_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
pushd $GITHUB_WORKSPACE/transformers/tests
export GIT_TEST="true"
make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all
pytest -v test_keras_transformer.py
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all
pytest -v test_keras_transformer.py
popd
- name: Build and Test FACE_DETECTION
if: ${{ env.FACE_DETECTION_ENABLE == 'true' }}
run: |
docker system prune -a -f --volumes
# build image
pushd $GITHUB_WORKSPACE/transformers/face_detection
echo ${{ secrets.KAGGLE_CREDS_JSON }} > kaggle_creds.json
export GIT_TEST="true"
make -B all
popd
# test image
kaggle datasets download -d sambitmukherjee/caffe-face-detector-opencv-pretrained-model
unzip caffe-face-detector-opencv-pretrained-model.zip -d $GITHUB_WORKSPACE/transformers/face_detection/model/
pushd $GITHUB_WORKSPACE/transformers/tests
pytest -o log_cli=true --log-cli-level=INFO test_face_detection.py
popd
export GIT_TEST="false"
make -B -C $GITHUB_WORKSPACE/transformers/keras_preprocess all
env:
KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }}