diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f712307b6..80deb83e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,16 @@ on: - '.gitignore' - 'docs/**' workflow_dispatch: + inputs: + manual_revision_reference: + required: false + type: string + manual_revision_test: + required: false + type: string + +env: + REVISION_REFERENCE: 9d31b2ec4df6d8228f370ff20c8267ec6ba39383 # v2.7.0 + pretrained_hf param jobs: Tests: @@ -32,7 +42,11 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.manual_revision_test }} - name: Set up Python ${{ matrix.python }} + id: pythonsetup uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -41,12 +55,12 @@ jobs: uses: actions/cache@v3 with: path: .env - key: venv-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('requirements*') }} - - name: Pytest durations + key: venv-${{ matrix.os }}-${{ steps.pythonsetup.outputs.python-version }}-${{ hashFiles('requirements*') }} + - name: Pytest durations cache uses: actions/cache@v3 with: path: .test_durations - key: test_durations-${{ matrix.os }}-${{ matrix.python }}-${{ matrix.job }}-${{ github.run_id }} + key: test_durations-${{ matrix.os }}-${{ steps.pythonsetup.outputs.python-version }}-${{ matrix.job }}-${{ github.run_id }} restore-keys: test_durations-0- - name: Setup if: steps.venv-cache.outputs.cache-hit != 'true' @@ -56,16 +70,49 @@ jobs: make install make install-test make install-training + - name: Prepare test data + run: | + source .env/bin/activate + python -m pytest \ + --quiet --co \ + --splitting-algorithm least_duration \ + --splits ${{ matrix.job_num }} \ + --group ${{ matrix.job }} \ + -m regression_test \ + tests \ + | head -n -2 | grep -Po 'test_inference_with_data\[\K[^]]*(?=])' \ + > models_gh_runner.txt + if [ -n "${{ inputs.manual_revision_reference }}" ]; then + REVISION_REFERENCE=${{ inputs.manual_revision_reference }} + fi + python tests/util_test.py \ + --save_model_list models_gh_runner.txt \ + --model_list models_gh_runner.txt \ + --git_revision $REVISION_REFERENCE - name: Unit tests run: | source .env/bin/activate + touch .test_durations + cp .test_durations durations_1 + mv .test_durations durations_2 python -m pytest \ -x -s -v \ --splitting-algorithm least_duration \ --splits ${{ matrix.job_num }} \ --group ${{ matrix.job }} \ --store-durations \ + --durations-path durations_1 \ + --clean-durations \ + -m "not regression_test" \ + tests + OPEN_CLIP_TEST_REG_MODELS=models_gh_runner.txt python -m pytest \ + -x -s -v \ + --store-durations \ + --durations-path durations_2 \ + --clean-durations \ + -m "regression_test" \ tests + jq -s -S 'add' durations_* > .test_durations - name: Collect pytest durations uses: actions/upload-artifact@v3 with: diff --git a/.gitignore b/.gitignore index 207834154..960651015 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ wandb/ models/ features/ results/ + +tests/data/ *.pt # Byte-compiled / optimized / DLL files diff --git a/README.md b/README.md index 40db96d32..3cb7db68d 100644 --- a/README.md +++ b/README.md @@ -111,15 +111,26 @@ If you want to make changes to contribute code, you can close openclip then run Install pip PyTorch as per https://pytorch.org/get-started/locally/ +You may run `make install-training` to install training deps + +#### Testing Test can be run with `make install-test` then `make test` `python -m pytest -x -s -v tests -k "training"` to run a specific test -When introducing new models, `python tests/util_test.py --model=xlm-roberta-large-ViT-H-14` can generate new output expected data. - -You may run `make install-training` to install training deps +Running regression tests against a specific git revision or tag: +1. Generate testing data + ```sh + python tests/util_test.py --model RN50 RN101 --save_model_list models.txt --git_revision 9d31b2ec4df6d8228f370ff20c8267ec6ba39383 + ``` + **_WARNING_: This will invoke git and modify your working tree, but will reset it to the current state after data has been generated! \ + Don't modify your working tree while test data is being generated this way.** +2. Run regression tests + ```sh + OPEN_CLIP_TEST_REG_MODELS=models.txt python -m pytest -x -s -v -m regression_test + ``` ### Sample single-process running code: @@ -513,7 +524,7 @@ quantifies robustness as accuracy beyond this baseline, i.e., how far a model li Even though the CLIP models trained with this codebase achieve much lower accuracy than those trained by OpenAI, our models still lie on the same trend of improved effective robustness (the purple line). Therefore, we can study what makes -CLIP robust without requiring industrial-scale compute. +CLIP robust without requiring industrial-scale compute. For more information on effective robustness, please see: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..9546b10ce --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + regression_test diff --git a/tests/data/input/random_image_224_224.pt b/tests/data/input/random_image_224_224.pt deleted file mode 100644 index 9c9e26d6e..000000000 Binary files a/tests/data/input/random_image_224_224.pt and /dev/null differ diff --git a/tests/data/input/random_image_240_240.pt b/tests/data/input/random_image_240_240.pt deleted file mode 100644 index a5d6d527c..000000000 Binary files a/tests/data/input/random_image_240_240.pt and /dev/null differ diff --git a/tests/data/input/random_image_256_256.pt b/tests/data/input/random_image_256_256.pt deleted file mode 100644 index 0be124820..000000000 Binary files a/tests/data/input/random_image_256_256.pt and /dev/null differ diff --git a/tests/data/input/random_image_280_280.pt b/tests/data/input/random_image_280_280.pt deleted file mode 100644 index 84730dcc2..000000000 Binary files a/tests/data/input/random_image_280_280.pt and /dev/null differ diff --git a/tests/data/input/random_image_288_288.pt b/tests/data/input/random_image_288_288.pt deleted file mode 100644 index 3bdeed450..000000000 Binary files a/tests/data/input/random_image_288_288.pt and /dev/null differ diff --git a/tests/data/input/random_image_320_320.pt b/tests/data/input/random_image_320_320.pt deleted file mode 100644 index 4dccbbcdd..000000000 Binary files a/tests/data/input/random_image_320_320.pt and /dev/null differ diff --git a/tests/data/input/random_image_336_336.pt b/tests/data/input/random_image_336_336.pt deleted file mode 100644 index b9c2405ac..000000000 Binary files a/tests/data/input/random_image_336_336.pt and /dev/null differ diff --git a/tests/data/input/random_image_384_384.pt b/tests/data/input/random_image_384_384.pt deleted file mode 100644 index 34c358765..000000000 Binary files a/tests/data/input/random_image_384_384.pt and /dev/null differ diff --git a/tests/data/input/random_image_448_448.pt b/tests/data/input/random_image_448_448.pt deleted file mode 100644 index 33eb374e5..000000000 Binary files a/tests/data/input/random_image_448_448.pt and /dev/null differ diff --git a/tests/data/input/random_text.pt b/tests/data/input/random_text.pt deleted file mode 100644 index 8a6458f2f..000000000 Binary files a/tests/data/input/random_text.pt and /dev/null differ diff --git a/tests/data/output/RN101-quickgelu_None_fp32_random_image.pt b/tests/data/output/RN101-quickgelu_None_fp32_random_image.pt deleted file mode 100644 index 7727a217e..000000000 Binary files a/tests/data/output/RN101-quickgelu_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN101-quickgelu_None_fp32_random_text.pt b/tests/data/output/RN101-quickgelu_None_fp32_random_text.pt deleted file mode 100644 index c1019fd4b..000000000 Binary files a/tests/data/output/RN101-quickgelu_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN101_None_fp32_random_image.pt b/tests/data/output/RN101_None_fp32_random_image.pt deleted file mode 100644 index 7727a217e..000000000 Binary files a/tests/data/output/RN101_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN101_None_fp32_random_text.pt b/tests/data/output/RN101_None_fp32_random_text.pt deleted file mode 100644 index fa6e3b939..000000000 Binary files a/tests/data/output/RN101_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN50-quickgelu_None_fp32_random_image.pt b/tests/data/output/RN50-quickgelu_None_fp32_random_image.pt deleted file mode 100644 index a4981cca4..000000000 Binary files a/tests/data/output/RN50-quickgelu_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN50-quickgelu_None_fp32_random_text.pt b/tests/data/output/RN50-quickgelu_None_fp32_random_text.pt deleted file mode 100644 index ea7b819d6..000000000 Binary files a/tests/data/output/RN50-quickgelu_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN50_None_fp32_random_image.pt b/tests/data/output/RN50_None_fp32_random_image.pt deleted file mode 100644 index a4981cca4..000000000 Binary files a/tests/data/output/RN50_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN50_None_fp32_random_text.pt b/tests/data/output/RN50_None_fp32_random_text.pt deleted file mode 100644 index a0c4adac2..000000000 Binary files a/tests/data/output/RN50_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN50x16_None_fp32_random_image.pt b/tests/data/output/RN50x16_None_fp32_random_image.pt deleted file mode 100644 index 9c69bac4b..000000000 Binary files a/tests/data/output/RN50x16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN50x16_None_fp32_random_text.pt b/tests/data/output/RN50x16_None_fp32_random_text.pt deleted file mode 100644 index 8d72fb8b1..000000000 Binary files a/tests/data/output/RN50x16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN50x4_None_fp32_random_image.pt b/tests/data/output/RN50x4_None_fp32_random_image.pt deleted file mode 100644 index e3ae62aea..000000000 Binary files a/tests/data/output/RN50x4_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN50x4_None_fp32_random_text.pt b/tests/data/output/RN50x4_None_fp32_random_text.pt deleted file mode 100644 index d69989b05..000000000 Binary files a/tests/data/output/RN50x4_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/RN50x64_None_fp32_random_image.pt b/tests/data/output/RN50x64_None_fp32_random_image.pt deleted file mode 100644 index 81f6f293b..000000000 Binary files a/tests/data/output/RN50x64_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/RN50x64_None_fp32_random_text.pt b/tests/data/output/RN50x64_None_fp32_random_text.pt deleted file mode 100644 index 6349e30b4..000000000 Binary files a/tests/data/output/RN50x64_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16-plus-240_None_fp32_random_image.pt b/tests/data/output/ViT-B-16-plus-240_None_fp32_random_image.pt deleted file mode 100644 index 1bec81c31..000000000 Binary files a/tests/data/output/ViT-B-16-plus-240_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16-plus-240_None_fp32_random_text.pt b/tests/data/output/ViT-B-16-plus-240_None_fp32_random_text.pt deleted file mode 100644 index 3c48155c5..000000000 Binary files a/tests/data/output/ViT-B-16-plus-240_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16-plus_None_fp32_random_image.pt b/tests/data/output/ViT-B-16-plus_None_fp32_random_image.pt deleted file mode 100644 index 07c5fb510..000000000 Binary files a/tests/data/output/ViT-B-16-plus_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16-plus_None_fp32_random_text.pt b/tests/data/output/ViT-B-16-plus_None_fp32_random_text.pt deleted file mode 100644 index fca83945e..000000000 Binary files a/tests/data/output/ViT-B-16-plus_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16_None_fp32_random_image.pt b/tests/data/output/ViT-B-16_None_fp32_random_image.pt deleted file mode 100644 index ca79d63d0..000000000 Binary files a/tests/data/output/ViT-B-16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-16_None_fp32_random_text.pt b/tests/data/output/ViT-B-16_None_fp32_random_text.pt deleted file mode 100644 index 6ccb5cb17..000000000 Binary files a/tests/data/output/ViT-B-16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32-plus-256_None_fp32_random_image.pt b/tests/data/output/ViT-B-32-plus-256_None_fp32_random_image.pt deleted file mode 100644 index f5ab99baf..000000000 Binary files a/tests/data/output/ViT-B-32-plus-256_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32-plus-256_None_fp32_random_text.pt b/tests/data/output/ViT-B-32-plus-256_None_fp32_random_text.pt deleted file mode 100644 index 5cbdf326e..000000000 Binary files a/tests/data/output/ViT-B-32-plus-256_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_image.pt b/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_image.pt deleted file mode 100644 index 070111a67..000000000 Binary files a/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_text.pt b/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_text.pt deleted file mode 100644 index 4b1a8ff77..000000000 Binary files a/tests/data/output/ViT-B-32-quickgelu_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32_None_fp32_random_image.pt b/tests/data/output/ViT-B-32_None_fp32_random_image.pt deleted file mode 100644 index ae01c43b9..000000000 Binary files a/tests/data/output/ViT-B-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-B-32_None_fp32_random_text.pt b/tests/data/output/ViT-B-32_None_fp32_random_text.pt deleted file mode 100644 index 3b914171a..000000000 Binary files a/tests/data/output/ViT-B-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-H-14_None_fp32_random_image.pt b/tests/data/output/ViT-H-14_None_fp32_random_image.pt deleted file mode 100644 index 53a20b462..000000000 Binary files a/tests/data/output/ViT-H-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-H-14_None_fp32_random_text.pt b/tests/data/output/ViT-H-14_None_fp32_random_text.pt deleted file mode 100644 index 851cb3d99..000000000 Binary files a/tests/data/output/ViT-H-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-H-16_None_fp32_random_image.pt b/tests/data/output/ViT-H-16_None_fp32_random_image.pt deleted file mode 100644 index e7297cf50..000000000 Binary files a/tests/data/output/ViT-H-16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-H-16_None_fp32_random_text.pt b/tests/data/output/ViT-H-16_None_fp32_random_text.pt deleted file mode 100644 index 814ae78ef..000000000 Binary files a/tests/data/output/ViT-H-16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14-280_None_fp32_random_image.pt b/tests/data/output/ViT-L-14-280_None_fp32_random_image.pt deleted file mode 100644 index 577c2b834..000000000 Binary files a/tests/data/output/ViT-L-14-280_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14-280_None_fp32_random_text.pt b/tests/data/output/ViT-L-14-280_None_fp32_random_text.pt deleted file mode 100644 index 6e7fbbb45..000000000 Binary files a/tests/data/output/ViT-L-14-280_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14-336_None_fp32_random_image.pt b/tests/data/output/ViT-L-14-336_None_fp32_random_image.pt deleted file mode 100644 index 6288800a0..000000000 Binary files a/tests/data/output/ViT-L-14-336_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14-336_None_fp32_random_text.pt b/tests/data/output/ViT-L-14-336_None_fp32_random_text.pt deleted file mode 100644 index ac2a046c0..000000000 Binary files a/tests/data/output/ViT-L-14-336_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14_None_fp32_random_image.pt b/tests/data/output/ViT-L-14_None_fp32_random_image.pt deleted file mode 100644 index baef22adf..000000000 Binary files a/tests/data/output/ViT-L-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-14_None_fp32_random_text.pt b/tests/data/output/ViT-L-14_None_fp32_random_text.pt deleted file mode 100644 index 1a1630839..000000000 Binary files a/tests/data/output/ViT-L-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-16-320_None_fp32_random_image.pt b/tests/data/output/ViT-L-16-320_None_fp32_random_image.pt deleted file mode 100644 index 08a75e27f..000000000 Binary files a/tests/data/output/ViT-L-16-320_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-16-320_None_fp32_random_text.pt b/tests/data/output/ViT-L-16-320_None_fp32_random_text.pt deleted file mode 100644 index 4de20391e..000000000 Binary files a/tests/data/output/ViT-L-16-320_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-16_None_fp32_random_image.pt b/tests/data/output/ViT-L-16_None_fp32_random_image.pt deleted file mode 100644 index a9a8261d0..000000000 Binary files a/tests/data/output/ViT-L-16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-L-16_None_fp32_random_text.pt b/tests/data/output/ViT-L-16_None_fp32_random_text.pt deleted file mode 100644 index 85bbf3056..000000000 Binary files a/tests/data/output/ViT-L-16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-16-alt_None_fp32_random_image.pt b/tests/data/output/ViT-M-16-alt_None_fp32_random_image.pt deleted file mode 100644 index a9c61f9c1..000000000 Binary files a/tests/data/output/ViT-M-16-alt_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-16-alt_None_fp32_random_text.pt b/tests/data/output/ViT-M-16-alt_None_fp32_random_text.pt deleted file mode 100644 index 025fc3f75..000000000 Binary files a/tests/data/output/ViT-M-16-alt_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-16_None_fp32_random_image.pt b/tests/data/output/ViT-M-16_None_fp32_random_image.pt deleted file mode 100644 index 490e43272..000000000 Binary files a/tests/data/output/ViT-M-16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-16_None_fp32_random_text.pt b/tests/data/output/ViT-M-16_None_fp32_random_text.pt deleted file mode 100644 index 19b595405..000000000 Binary files a/tests/data/output/ViT-M-16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-32-alt_None_fp32_random_image.pt b/tests/data/output/ViT-M-32-alt_None_fp32_random_image.pt deleted file mode 100644 index 2ea855d0d..000000000 Binary files a/tests/data/output/ViT-M-32-alt_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-32-alt_None_fp32_random_text.pt b/tests/data/output/ViT-M-32-alt_None_fp32_random_text.pt deleted file mode 100644 index c884c0b34..000000000 Binary files a/tests/data/output/ViT-M-32-alt_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-32_None_fp32_random_image.pt b/tests/data/output/ViT-M-32_None_fp32_random_image.pt deleted file mode 100644 index 1d8305feb..000000000 Binary files a/tests/data/output/ViT-M-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-M-32_None_fp32_random_text.pt b/tests/data/output/ViT-M-32_None_fp32_random_text.pt deleted file mode 100644 index f3a514c81..000000000 Binary files a/tests/data/output/ViT-M-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-16-alt_None_fp32_random_image.pt b/tests/data/output/ViT-S-16-alt_None_fp32_random_image.pt deleted file mode 100644 index 8dc9f73d0..000000000 Binary files a/tests/data/output/ViT-S-16-alt_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-16-alt_None_fp32_random_text.pt b/tests/data/output/ViT-S-16-alt_None_fp32_random_text.pt deleted file mode 100644 index bd5d03b75..000000000 Binary files a/tests/data/output/ViT-S-16-alt_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-16_None_fp32_random_image.pt b/tests/data/output/ViT-S-16_None_fp32_random_image.pt deleted file mode 100644 index e47292179..000000000 Binary files a/tests/data/output/ViT-S-16_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-16_None_fp32_random_text.pt b/tests/data/output/ViT-S-16_None_fp32_random_text.pt deleted file mode 100644 index e43e4a32c..000000000 Binary files a/tests/data/output/ViT-S-16_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-32-alt_None_fp32_random_image.pt b/tests/data/output/ViT-S-32-alt_None_fp32_random_image.pt deleted file mode 100644 index 21d2dffe4..000000000 Binary files a/tests/data/output/ViT-S-32-alt_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-32-alt_None_fp32_random_text.pt b/tests/data/output/ViT-S-32-alt_None_fp32_random_text.pt deleted file mode 100644 index 6e209964c..000000000 Binary files a/tests/data/output/ViT-S-32-alt_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-32_None_fp32_random_image.pt b/tests/data/output/ViT-S-32_None_fp32_random_image.pt deleted file mode 100644 index 708292967..000000000 Binary files a/tests/data/output/ViT-S-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-S-32_None_fp32_random_text.pt b/tests/data/output/ViT-S-32_None_fp32_random_text.pt deleted file mode 100644 index 97847c6e0..000000000 Binary files a/tests/data/output/ViT-S-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-bigG-14_None_fp32_random_image.pt b/tests/data/output/ViT-bigG-14_None_fp32_random_image.pt deleted file mode 100644 index adfa84575..000000000 Binary files a/tests/data/output/ViT-bigG-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-bigG-14_None_fp32_random_text.pt b/tests/data/output/ViT-bigG-14_None_fp32_random_text.pt deleted file mode 100644 index c999c5f72..000000000 Binary files a/tests/data/output/ViT-bigG-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-e-14_None_fp32_random_image.pt b/tests/data/output/ViT-e-14_None_fp32_random_image.pt deleted file mode 100644 index 6f774de45..000000000 Binary files a/tests/data/output/ViT-e-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-e-14_None_fp32_random_text.pt b/tests/data/output/ViT-e-14_None_fp32_random_text.pt deleted file mode 100644 index d76799b59..000000000 Binary files a/tests/data/output/ViT-e-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/ViT-g-14_None_fp32_random_image.pt b/tests/data/output/ViT-g-14_None_fp32_random_image.pt deleted file mode 100644 index e4521e984..000000000 Binary files a/tests/data/output/ViT-g-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/ViT-g-14_None_fp32_random_text.pt b/tests/data/output/ViT-g-14_None_fp32_random_text.pt deleted file mode 100644 index dde0cee6b..000000000 Binary files a/tests/data/output/ViT-g-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_image.pt b/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_image.pt deleted file mode 100644 index fb5fd7218..000000000 Binary files a/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_text.pt b/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_text.pt deleted file mode 100644 index 21b077347..000000000 Binary files a/tests/data/output/mt5-base-ViT-B-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_image.pt b/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_image.pt deleted file mode 100644 index 58417542f..000000000 Binary files a/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_text.pt b/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_text.pt deleted file mode 100644 index 07b462e9c..000000000 Binary files a/tests/data/output/mt5-xl-ViT-H-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/roberta-ViT-B-32_None_fp32_random_image.pt b/tests/data/output/roberta-ViT-B-32_None_fp32_random_image.pt deleted file mode 100644 index 070111a67..000000000 Binary files a/tests/data/output/roberta-ViT-B-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/roberta-ViT-B-32_None_fp32_random_text.pt b/tests/data/output/roberta-ViT-B-32_None_fp32_random_text.pt deleted file mode 100644 index 2279f7e28..000000000 Binary files a/tests/data/output/roberta-ViT-B-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-convnext_base_None_fp32_random_image.pt b/tests/data/output/timm-convnext_base_None_fp32_random_image.pt deleted file mode 100644 index 57635a485..000000000 Binary files a/tests/data/output/timm-convnext_base_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-convnext_base_None_fp32_random_text.pt b/tests/data/output/timm-convnext_base_None_fp32_random_text.pt deleted file mode 100644 index fb519688c..000000000 Binary files a/tests/data/output/timm-convnext_base_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-convnext_large_None_fp32_random_image.pt b/tests/data/output/timm-convnext_large_None_fp32_random_image.pt deleted file mode 100644 index 2aa0c41a6..000000000 Binary files a/tests/data/output/timm-convnext_large_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-convnext_large_None_fp32_random_text.pt b/tests/data/output/timm-convnext_large_None_fp32_random_text.pt deleted file mode 100644 index 318a940b5..000000000 Binary files a/tests/data/output/timm-convnext_large_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_image.pt b/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_image.pt deleted file mode 100644 index 15730f0a9..000000000 Binary files a/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_text.pt b/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_text.pt deleted file mode 100644 index a07327299..000000000 Binary files a/tests/data/output/timm-efficientnetv2_rw_s_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-resnetaa50d_None_fp32_random_image.pt b/tests/data/output/timm-resnetaa50d_None_fp32_random_image.pt deleted file mode 100644 index 0659d71cb..000000000 Binary files a/tests/data/output/timm-resnetaa50d_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-resnetaa50d_None_fp32_random_text.pt b/tests/data/output/timm-resnetaa50d_None_fp32_random_text.pt deleted file mode 100644 index de0b26ae9..000000000 Binary files a/tests/data/output/timm-resnetaa50d_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_image.pt b/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_image.pt deleted file mode 100644 index b2507c238..000000000 Binary files a/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_text.pt b/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_text.pt deleted file mode 100644 index 27e0821ef..000000000 Binary files a/tests/data/output/timm-swin_base_patch4_window7_224_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_image.pt b/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_image.pt deleted file mode 100644 index d20b5ef19..000000000 Binary files a/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_text.pt b/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_text.pt deleted file mode 100644 index 7d389ecae..000000000 Binary files a/tests/data/output/timm-vit_relpos_medium_patch16_cls_224_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_image.pt b/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_image.pt deleted file mode 100644 index ae01c43b9..000000000 Binary files a/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_text.pt b/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_text.pt deleted file mode 100644 index d9e5fd1f0..000000000 Binary files a/tests/data/output/xlm-roberta-base-ViT-B-32_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_image.pt b/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_image.pt deleted file mode 100644 index 2deb20d20..000000000 Binary files a/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_image.pt and /dev/null differ diff --git a/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_text.pt b/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_text.pt deleted file mode 100644 index 4212cd143..000000000 Binary files a/tests/data/output/xlm-roberta-large-ViT-H-14_None_fp32_random_text.pt and /dev/null differ diff --git a/tests/test_inference.py b/tests/test_inference.py index f1409421b..0b3fdc1e0 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -1,17 +1,16 @@ import os -import random import pytest -import numpy import torch -from PIL import Image import open_clip import util_test os.environ['CUDA_VISIBLE_DEVICES'] = '' -# test all model with some exceptions -models_to_test = set(open_clip.list_models()).difference({ +models_to_test = set(open_clip.list_models()) + +# testing excemptions +models_to_test = models_to_test.difference({ # not available with timm yet # see https://github.com/mlfoundations/open_clip/issues/219 'timm-convnext_xlarge', @@ -22,15 +21,24 @@ 'mt5-xl-ViT-H-14', }) +if 'OPEN_CLIP_TEST_REG_MODELS' in os.environ: + external_model_list = os.environ['OPEN_CLIP_TEST_REG_MODELS'] + with open(external_model_list, 'r') as f: + models_to_test = set(f.read().splitlines()).intersection(models_to_test) + print(f"Selected models from {external_model_list}: {models_to_test}") + +models_to_test = list(models_to_test) +models_to_test.sort() + +@pytest.mark.regression_test @pytest.mark.parametrize('model_name', models_to_test) def test_inference_with_data( model_name, pretrained = None, + pretrained_hf = False, precision = 'fp32', jit = False, force_quick_gelu = False, - # experimentally determined between author machine and GH runner - tolerance = torch.finfo(torch.float32).resolution * 4 ): util_test.seed_all() model, _, preprocess_val = open_clip.create_model_and_transforms( @@ -39,30 +47,34 @@ def test_inference_with_data( precision = precision, jit = jit, force_quick_gelu = force_quick_gelu, - pretrained_hf = False + pretrained_hf = pretrained_hf ) - model_id = f'{model_name}_{pretrained}_{precision}' + model_id = f'{model_name}_{pretrained or pretrained_hf}_{precision}' input_dir, output_dir = util_test.get_data_dirs() # text input_text_path = os.path.join(input_dir, 'random_text.pt') gt_text_path = os.path.join(output_dir, f'{model_id}_random_text.pt') - assert os.path.isfile(input_text_path), f"missing test data, expected at {input_text_path}" - assert os.path.isfile(gt_text_path), f"missing test data, expected at {gt_text_path}" + if not os.path.isfile(input_text_path): + pytest.skip(reason = f"missing test data, expected at {input_text_path}") + if not os.path.isfile(gt_text_path): + pytest.skip(reason = f"missing test data, expected at {gt_text_path}") input_text = torch.load(input_text_path) gt_text = torch.load(gt_text_path) y_text = util_test.inference_text(model, model_name, input_text) - assert torch.allclose(y_text, gt_text, atol=tolerance), f"text output differs @ {input_text_path}" + assert (y_text == gt_text).all(), f"text output differs @ {input_text_path}" # image image_size = model.visual.image_size if not isinstance(image_size, tuple): image_size = (image_size, image_size) input_image_path = os.path.join(input_dir, f'random_image_{image_size[0]}_{image_size[1]}.pt') gt_image_path = os.path.join(output_dir, f'{model_id}_random_image.pt') - assert os.path.isfile(input_image_path), f"missing test data, expected at {input_image_path}" - assert os.path.isfile(gt_image_path), f"missing test data, expected at {gt_image_path}" + if not os.path.isfile(input_image_path): + pytest.skip(reason = f"missing test data, expected at {input_image_path}") + if not os.path.isfile(gt_image_path): + pytest.skip(reason = f"missing test data, expected at {gt_image_path}") input_image = torch.load(input_image_path) gt_image = torch.load(gt_image_path) y_image = util_test.inference_image(model, preprocess_val, input_image) - assert torch.allclose(y_image, gt_image, atol=tolerance), f"image output differs @ {input_image_path}" + assert (y_image == gt_image).all(), f"image output differs @ {input_image_path}" diff --git a/tests/util_test.py b/tests/util_test.py index 5393f6c52..b2a2c9c3d 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -1,10 +1,11 @@ - import os import random import numpy as np from PIL import Image import torch -import open_clip + +if __name__ != '__main__': + import open_clip os.environ['CUDA_VISIBLE_DEVICES'] = '' @@ -93,13 +94,14 @@ def create_test_data_for_model( pretrained = None, precision = 'fp32', jit = False, + pretrained_hf = False, force_quick_gelu = False, create_missing_input_data = True, batches = 1, batch_size = 1, overwrite = False ): - model_id = f'{model_name}_{pretrained}_{precision}' + model_id = f'{model_name}_{pretrained or pretrained_hf}_{precision}' input_dir, output_dir = get_data_dirs() output_file_text = os.path.join(output_dir, f'{model_id}_random_text.pt') output_file_image = os.path.join(output_dir, f'{model_id}_random_image.pt') @@ -114,7 +116,7 @@ def create_test_data_for_model( precision = precision, jit = jit, force_quick_gelu = force_quick_gelu, - pretrained_hf = False + pretrained_hf = pretrained_hf ) # text if overwrite or not text_exists: @@ -155,66 +157,128 @@ def create_test_data( batch_size = 1, overwrite = False ): - models = set(models).difference({ + models = list(set(models).difference({ # not available with timm # see https://github.com/mlfoundations/open_clip/issues/219 'timm-convnext_xlarge', 'timm-vit_medium_patch16_gap_256' - }) + }).intersection(open_clip.list_models())) + models.sort() + print(f"generating test data for:\n{models}") for model_name in models: + print(model_name) create_test_data_for_model( model_name, batches = batches, batch_size = batch_size, overwrite = overwrite ) + return models +def _sytem_assert(string): + assert os.system(string) == 0 def main(args): + global open_clip + import importlib + import shutil + import subprocess import argparse - parser = argparse.ArgumentParser(description="Populate test data directory") + parser = argparse.ArgumentParser(description = "Populate test data directory") + parser.add_argument( + '-a', '--all', + action = 'store_true', + help = "create test data for all models" + ) + parser.add_argument( + '-m', '--model', + type = str, + default = [], + nargs = '+', + help = "model(s) to create test data for" + ) parser.add_argument( - "--all", - default=False, - action='store_true', - help="create test data for all models" + '-f', '--model_list', + type = str, + help = "path to a text file containing a list of model names, one model per line" ) parser.add_argument( - "--model", - default=None, - type=str, - help="model to create test data for (default: None)" + '-s', '--save_model_list', + type = str, + help = "path to save the list of models that data was generated for" ) parser.add_argument( - "--overwrite", - default=False, - action='store_true', - help="overwrite existing data" + '-g', '--git_revision', + type = str, + help = "git revision to generate test data for" ) parser.add_argument( - "--num_batches", - default=1, - type=int, - help="amount of data batches to create (default: 1)" + '--overwrite', + action = 'store_true', + help = "overwrite existing output data" ) parser.add_argument( - "--batch_size", - default=1, - type=int, - help="test data batch size (default: 1)" + '-n', '--num_batches', + default = 1, + type = int, + help = "amount of data batches to create (default: 1)" + ) + parser.add_argument( + '-b', '--batch_size', + default = 1, + type = int, + help = "test data batch size (default: 1)" ) args = parser.parse_args(args) - if not args.all and args.model is None: + model_list = [] + if args.model_list is not None: + with open(args.model_list, 'r') as f: + model_list = f.read().splitlines() + if not args.all and len(args.model) < 1 and len(model_list) < 1: + print("error: at least one model name is required") parser.print_help() - parser.exit() - models = open_clip.list_models() if args.all else [args.model] - print(f"generating test data for:\n{models}") - create_test_data( - models, - batches = args.num_batches, - batch_size = args.batch_size, - overwrite = args.overwrite - ) + parser.exit(1) + if args.git_revision is not None: + stash_output = subprocess.check_output(['git', 'stash']).decode().splitlines() + has_stash = len(stash_output) > 0 and stash_output[0] != 'No local changes to save' + current_branch = subprocess.check_output(['git', 'branch', '--show-current']) + if len(current_branch) < 1: + # not on a branch -> detached head + current_branch = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + current_branch = current_branch.splitlines()[0].decode() + try: + _sytem_assert(f'git checkout {args.git_revision}') + except AssertionError as e: + _sytem_assert(f'git checkout -f {current_branch}') + if has_stash: + os.system(f'git stash pop') + raise e + open_clip = importlib.import_module('open_clip') + models = open_clip.list_models() if args.all else args.model + model_list + try: + models = create_test_data( + models, + batches = args.num_batches, + batch_size = args.batch_size, + overwrite = args.overwrite + ) + finally: + if args.git_revision is not None: + test_dir = os.path.join(os.path.dirname(__file__), 'data') + test_dir_ref = os.path.join(os.path.dirname(__file__), 'data_ref') + if os.path.exists(test_dir_ref): + shutil.rmtree(test_dir_ref, ignore_errors = True) + if os.path.exists(test_dir): + os.rename(test_dir, test_dir_ref) + _sytem_assert(f'git checkout {current_branch}') + if has_stash: + os.system(f'git stash pop') + os.rename(test_dir_ref, test_dir) + if args.save_model_list is not None: + print(f"Saving model list as {args.save_model_list}") + with open(args.save_model_list, 'w') as f: + for m in models: + print(m, file=f) if __name__ == '__main__':