Parallelize CI tests (#228)

* parallelize CI tests

* disable macos testing for now, as it's slow

* typo

* util_test only import argparse if called as script

.github/workflows/ci.yml

@@ -4,31 +4,90 @@ on:
   push:
     branches:
     - main
+    paths-ignore:
+    - '**.md'
+    - 'CITATION.cff'
+    - 'LICENSE'
+    - '.gitignore'
+    - 'docs/**'
   pull_request:
     branches:
     - main
+    paths-ignore:
+    - '**.md'
+    - 'CITATION.cff'
+    - 'LICENSE'
+    - '.gitignore'
+    - 'docs/**'
+  workflow_dispatch:
 
 jobs:
-  tests:
-    runs-on: ubuntu-latest
+  Tests:
     strategy:
       matrix:
-        python-version: [3.8]
-
+        os: [ ubuntu-latest ] #, macos-latest ]
+        python: [ 3.8 ]
+        job_num: [ 4 ]
+        job: [ 1, 2, 3, 4 ]
+    runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Venv cache
+      id: venv-cache
+      uses: actions/cache@v3
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install
+        path: .env
+        key: venv-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('requirements*') }}
+    - name: Pytest durations
+      uses: actions/cache@v3
+      with:
+        path: .test_durations
+        key: test_durations-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.job }}-${{ github.run_id }}
+        restore-keys: test_durations-0-
+    - name: Setup
+      if: steps.venv-cache.outputs.cache-hit != 'true'
       run: |
         python3 -m venv .env
         source .env/bin/activate
         make install
-        make install-dev
+        make install-test
         make install-training
     - name: Unit tests
       run: |
         source .env/bin/activate
-        make test
+        python -m pytest \
+          -x -s -v \
+          --splitting-algorithm least_duration \
+          --splits ${{ matrix.job_num }} \
+          --group ${{ matrix.job }} \
+          --store-durations \
+          tests
+    - name: Collect pytest durations
+      uses: actions/upload-artifact@v3
+      with:
+        name: pytest_durations_${{ matrix.os }}-${{ matrix.python }}-${{ matrix.job }}
+        path: .test_durations
+
+  Collect:
+    needs: Tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: .test_durations
+        key: test_durations-0-${{ github.run_id }}
+    - name: Collect
+      uses: actions/download-artifact@v3
+      with:
+        path: artifacts
+    - name: Consolidate
+      run: |
+        jq -n -S \
+          'reduce (inputs | to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \
+          artifacts/pytest_durations_*/.test_durations > .test_durations
+

.github/workflows/clear-cache.yml

@@ -0,0 +1,29 @@
+name: Clear cache
+
+on:
+  workflow_dispatch:
+
+permissions:
+  actions: write
+
+jobs:
+  clear-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clear cache
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const caches = await github.rest.actions.getActionsCacheList({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            for (const cache of caches.data.actions_caches) {
+              console.log(cache)
+              await github.rest.actions.deleteActionsCacheById({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                cache_id: cache.id,
+              })
+            }
+

Makefile

@@ -5,7 +5,7 @@ install: ## [Local development] Upgrade pip, install requirements, install packa
 install-training:
 	python -m pip install -r requirements-training.txt
 
-install-dev: ## [Local development] Install test requirements
+install-test: ## [Local development] Install test requirements
 	python -m pip install -r requirements-test.txt
 
 test: ## [Local development] Run unit tests

README.md

@@ -120,7 +120,7 @@ Install pip PyTorch as per https://pytorch.org/get-started/locally/
 
 #### Tests
 
-Test can be run with `make install-dev` then `make test`
+Test can be run with `make install-test` then `make test`
 
 `python -m pytest -x -s -v tests -k "training"` to run a specific test
 

requirements-test.txt

@@ -1,4 +1,4 @@
-pytest-xdist==2.5.0
-pytest==7.0.1
+pytest-split==0.8.0
+pytest==7.2.0
 transformers
 timm==0.6.11

tests/test_training_simple.py

@@ -1,11 +1,14 @@
 
-import torch
+import os
+import sys
+import pytest
 from PIL import Image
+import torch
 from training.main import main
-import pytest
-import os
+
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 
+@pytest.mark.skipif(sys.platform.startswith('darwin'), reason="macos pickle bug with locals")
 def test_training():
     main([
     '--save-frequency', '1',
@@ -20,3 +23,4 @@ def test_training():
     '--workers', '2',
     '--model', 'RN50'
     ])
+

tests/util_test.py

@@ -5,12 +5,13 @@
 from PIL import Image
 import torch
 import open_clip
-import argparse
 
 os.environ['CUDA_VISIBLE_DEVICES'] = ''
 
 def seed_all(seed = 0):
     torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    torch.use_deterministic_algorithms(True, warn_only=False)
     random.seed(seed)
     np.random.seed(seed)
     torch.manual_seed(seed)
@@ -169,6 +170,7 @@ def create_test_data(
 
 
 def main(args):
+    import argparse
     parser = argparse.ArgumentParser(description="Populate test data directory")
     parser.add_argument(
         "--all",
@@ -218,4 +220,3 @@ def main(args):
     import sys
     main(sys.argv[1:])
 
-