From 86dca97ba4634a588b3db931f963bcd9093de6fe Mon Sep 17 00:00:00 2001 From: lopho Date: Thu, 17 Nov 2022 18:32:24 +0100 Subject: [PATCH] Parallelize CI tests (#228) * parallelize CI tests * disable macos testing for now, as it's slow * typo * util_test only import argparse if called as script --- .github/workflows/ci.yml | 81 ++++++++++++++++++++++++++----- .github/workflows/clear-cache.yml | 29 +++++++++++ Makefile | 2 +- README.md | 2 +- requirements-test.txt | 4 +- tests/test_training_simple.py | 10 ++-- tests/util_test.py | 5 +- 7 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/clear-cache.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b97da7a84..f712307b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,31 +4,90 @@ on: push: branches: - main + paths-ignore: + - '**.md' + - 'CITATION.cff' + - 'LICENSE' + - '.gitignore' + - 'docs/**' pull_request: branches: - main + paths-ignore: + - '**.md' + - 'CITATION.cff' + - 'LICENSE' + - '.gitignore' + - 'docs/**' + workflow_dispatch: jobs: - tests: - runs-on: ubuntu-latest + Tests: strategy: matrix: - python-version: [3.8] - + os: [ ubuntu-latest ] #, macos-latest ] + python: [ 3.8 ] + job_num: [ 4 ] + job: [ 1, 2, 3, 4 ] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Venv cache + id: venv-cache + uses: actions/cache@v3 with: - python-version: ${{ matrix.python-version }} - - name: Install + path: .env + key: venv-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('requirements*') }} + - name: Pytest durations + uses: actions/cache@v3 + with: + path: .test_durations + key: test_durations-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.job }}-${{ github.run_id }} + restore-keys: test_durations-0- + - name: Setup + if: steps.venv-cache.outputs.cache-hit != 'true' run: | python3 -m venv .env source .env/bin/activate make install - make install-dev + make install-test make install-training - name: Unit tests run: | source .env/bin/activate - make test + python -m pytest \ + -x -s -v \ + --splitting-algorithm least_duration \ + --splits ${{ matrix.job_num }} \ + --group ${{ matrix.job }} \ + --store-durations \ + tests + - name: Collect pytest durations + uses: actions/upload-artifact@v3 + with: + name: pytest_durations_${{ matrix.os }}-${{ matrix.python }}-${{ matrix.job }} + path: .test_durations + + Collect: + needs: Tests + runs-on: ubuntu-latest + steps: + - name: Cache + uses: actions/cache@v3 + with: + path: .test_durations + key: test_durations-0-${{ github.run_id }} + - name: Collect + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: Consolidate + run: | + jq -n -S \ + 'reduce (inputs | to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \ + artifacts/pytest_durations_*/.test_durations > .test_durations + diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml new file mode 100644 index 000000000..22a1a2461 --- /dev/null +++ b/.github/workflows/clear-cache.yml @@ -0,0 +1,29 @@ +name: Clear cache + +on: + workflow_dispatch: + +permissions: + actions: write + +jobs: + clear-cache: + runs-on: ubuntu-latest + steps: + - name: Clear cache + uses: actions/github-script@v6 + with: + script: | + const caches = await github.rest.actions.getActionsCacheList({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + for (const cache of caches.data.actions_caches) { + console.log(cache) + await github.rest.actions.deleteActionsCacheById({ + owner: context.repo.owner, + repo: context.repo.repo, + cache_id: cache.id, + }) + } + diff --git a/Makefile b/Makefile index 248e5e3b4..ff07eccef 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ install: ## [Local development] Upgrade pip, install requirements, install packa install-training: python -m pip install -r requirements-training.txt -install-dev: ## [Local development] Install test requirements +install-test: ## [Local development] Install test requirements python -m pip install -r requirements-test.txt test: ## [Local development] Run unit tests diff --git a/README.md b/README.md index 7ef9fe43b..3349daf8f 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Install pip PyTorch as per https://pytorch.org/get-started/locally/ #### Tests -Test can be run with `make install-dev` then `make test` +Test can be run with `make install-test` then `make test` `python -m pytest -x -s -v tests -k "training"` to run a specific test diff --git a/requirements-test.txt b/requirements-test.txt index 1ef4c1ca3..5d2e7e147 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -pytest-xdist==2.5.0 -pytest==7.0.1 +pytest-split==0.8.0 +pytest==7.2.0 transformers timm==0.6.11 diff --git a/tests/test_training_simple.py b/tests/test_training_simple.py index 9666a7e75..2970d3db0 100644 --- a/tests/test_training_simple.py +++ b/tests/test_training_simple.py @@ -1,11 +1,14 @@ -import torch +import os +import sys +import pytest from PIL import Image +import torch from training.main import main -import pytest -import os + os.environ["CUDA_VISIBLE_DEVICES"] = "" +@pytest.mark.skipif(sys.platform.startswith('darwin'), reason="macos pickle bug with locals") def test_training(): main([ '--save-frequency', '1', @@ -20,3 +23,4 @@ def test_training(): '--workers', '2', '--model', 'RN50' ]) + diff --git a/tests/util_test.py b/tests/util_test.py index fca030976..71044ed5a 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -5,12 +5,13 @@ from PIL import Image import torch import open_clip -import argparse os.environ['CUDA_VISIBLE_DEVICES'] = '' def seed_all(seed = 0): torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.use_deterministic_algorithms(True, warn_only=False) random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) @@ -169,6 +170,7 @@ def create_test_data( def main(args): + import argparse parser = argparse.ArgumentParser(description="Populate test data directory") parser.add_argument( "--all", @@ -218,4 +220,3 @@ def main(args): import sys main(sys.argv[1:]) -