feat(vectors): support sparse vectors in Milvus (#1362) #2804
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'release-**' | |
| pull_request: | |
| branches: | |
| - master | |
| - 'release-**' | |
| jobs: | |
| unit_test: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, ubuntu-24.04-arm] | |
| name: unit tests | |
| runs-on: ${{ matrix.os }} | |
| services: | |
| mysql: | |
| image: mysql:8.0 | |
| ports: | |
| - 3306 | |
| env: | |
| MYSQL_ROOT_PASSWORD: password | |
| options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
| postgres: | |
| image: postgres:10.0 | |
| ports: | |
| - 5432 | |
| env: | |
| POSTGRES_USER: gorse | |
| POSTGRES_PASSWORD: gorse_pass | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| mongo: | |
| image: mongo:6 | |
| ports: | |
| - 27017 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: root | |
| MONGO_INITDB_ROOT_PASSWORD: password | |
| options: >- | |
| --health-cmd mongosh | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| clickhouse: | |
| image: clickhouse/clickhouse-server:26.2 | |
| ports: | |
| - 8123 | |
| env: | |
| CLICKHOUSE_USER: gorse | |
| CLICKHOUSE_PASSWORD: gorse_pass | |
| options: >- | |
| --health-cmd="clickhouse-client --query 'SELECT 1'" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=5 | |
| redis: | |
| image: redis:8 | |
| ports: | |
| - 6379 | |
| rustfs: | |
| image: rustfs/rustfs:alpha | |
| ports: | |
| - 9000 | |
| env: | |
| RUSTFS_ACCESS_KEY: rustfsadmin | |
| RUSTFS_SECRET_KEY: rustfsadmin | |
| qdrant: | |
| image: qdrant/qdrant:latest | |
| ports: | |
| - 6334 | |
| weaviate: | |
| image: cr.weaviate.io/semitechnologies/weaviate:1.35.7 | |
| ports: | |
| - 8080 | |
| steps: | |
| - name: Set up dataset | |
| run: | | |
| mkdir -p ~/.gorse/dataset | |
| mkdir -p ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/mnist.zip -P ~/.gorse/download | |
| unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/mnist.zip -d ~/.gorse/dataset | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: Install and start Azurite | |
| uses: potatoqualitee/azuright@v1 | |
| - name: Setup Milvus | |
| run: | | |
| curl -sfL https://raw.githubusercontent.com/milvus-io/milvus/6da09b8b02b1/scripts/standalone_embed.sh -o standalone_embed.sh | |
| bash standalone_embed.sh start | |
| working-directory: ${{ runner.temp }} | |
| - name: Test | |
| run: go test -timeout 30m ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... | |
| env: | |
| # MySQL | |
| MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mysql.ports[3306] }})/ | |
| # Postgres | |
| POSTGRES_URI: postgres://gorse:gorse_pass@localhost:${{ job.services.postgres.ports[5432] }}/ | |
| # MongoDB | |
| MONGO_URI: mongodb://root:password@localhost:${{ job.services.mongo.ports[27017] }}/ | |
| # ClickHouse | |
| CLICKHOUSE_URI: clickhouse://gorse:gorse_pass@localhost:${{ job.services.clickhouse.ports[8123] }}/ | |
| # Redis | |
| REDIS_URI: redis://localhost:${{ job.services.redis.ports[6379] }}/ | |
| # S3 | |
| S3_ENDPOINT: localhost:${{ job.services.rustfs.ports[9000] }} | |
| S3_ACCESS_KEY_ID: rustfsadmin | |
| S3_SECRET_ACCESS_KEY: rustfsadmin | |
| # Azure Blob (Azurite) | |
| AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; | |
| # Qdrant | |
| QDRANT_URI: qdrant://localhost:${{ job.services.qdrant.ports[6334] }}/ | |
| # Weaviate | |
| WEAVIATE_URI: weaviate://localhost:${{ job.services.weaviate.ports[8080] }}/ | |
| - name: Upload coverage to Codecov | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| files: ./coverage.txt | |
| fail_ci_if_error: false | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| unit_test_macos: | |
| name: unit tests (macos-latest) | |
| runs-on: macos-latest | |
| steps: | |
| - name: Set up dataset | |
| run: | | |
| mkdir -p ~/.gorse/dataset | |
| mkdir -p ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download | |
| wget https://cdn.gorse.io/datasets/mnist.zip -P ~/.gorse/download | |
| unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset | |
| unzip ~/.gorse/download/mnist.zip -d ~/.gorse/dataset | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: Test | |
| run: go test -timeout 20m ./... | |
| unit_test_windows: | |
| strategy: | |
| matrix: | |
| os: [windows-latest, windows-11-arm] | |
| name: unit tests | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Set up dataset | |
| run: | | |
| New-Item -Type Directory -Path ~/.gorse/dataset | |
| New-Item -Type Directory -Path ~/.gorse/download | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/ml-100k.zip -OutFile ~/.gorse/download/ml-100k.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/ml-1m.zip -OutFile ~/.gorse/download/ml-1m.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/pinterest-20.zip -OutFile ~/.gorse/download/pinterest-20.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/frappe.zip -OutFile ~/.gorse/download/frappe.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/ml-tag.zip -OutFile ~/.gorse/download/ml-tag.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/criteo.zip -OutFile ~/.gorse/download/criteo.zip | |
| Invoke-WebRequest https://cdn.gorse.io/datasets/mnist.zip -OutFile ~/.gorse/download/mnist.zip | |
| Expand-Archive ~/.gorse/download/ml-100k.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/ml-1m.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/pinterest-20.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/frappe.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/ml-tag.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/criteo.zip -DestinationPath ~/.gorse/dataset | |
| Expand-Archive ~/.gorse/download/mnist.zip -DestinationPath ~/.gorse/dataset | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: Set up Windows ARM64 C toolchain | |
| if: matrix.os == 'windows-11-arm' | |
| id: msys2 | |
| uses: msys2/setup-msys2@v2 | |
| with: | |
| msystem: CLANGARM64 | |
| update: true | |
| install: mingw-w64-clang-aarch64-gcc-compat | |
| - name: Configure Windows ARM64 C compiler | |
| if: matrix.os == 'windows-11-arm' | |
| shell: pwsh | |
| run: | | |
| $cc = Join-Path '${{ steps.msys2.outputs.msys2-location }}' 'clangarm64\bin\gcc.exe' | |
| $target = & $cc -dumpmachine | |
| if ($LASTEXITCODE -ne 0) { | |
| exit $LASTEXITCODE | |
| } | |
| if ($target -ne 'aarch64-w64-windows-gnu') { | |
| Write-Error "Unexpected C compiler target: $target" | |
| exit 1 | |
| } | |
| "CC=$cc" >> $env:GITHUB_ENV | |
| - name: Test | |
| run: go test -timeout 20m ./... | |
| integrate_test: | |
| name: integrate tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| database: [mysql, postgres, mongo, sqlite] | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - uses: cuchi/jinja2-action@v1.2.0 | |
| with: | |
| template: client/docker-compose.yml.j2 | |
| output_file: docker-compose.yml | |
| strict: true | |
| variables: | | |
| database=${{ matrix.database }} | |
| - name: Setup test | |
| run: ./client/setup-test.sh | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| - name: Test | |
| run: go test ./client/ | |
| env: | |
| GORSE_SERVER_ENDPOINT: http://localhost:8087 | |
| GORSE_DASHBOARD_ENDPOINT: http://localhost:8088 | |
| compat_test: | |
| name: compatibility tests | |
| runs-on: ubuntu-latest | |
| services: | |
| mariadb: | |
| image: mariadb:10.2 | |
| ports: | |
| - 3306 | |
| env: | |
| MYSQL_ROOT_PASSWORD: password | |
| kvrocks: | |
| image: apache/kvrocks:nightly | |
| ports: | |
| - 6666 | |
| steps: | |
| - name: Install pre-requisites | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: qemu-user-static | |
| - name: Setup Redis cluster | |
| uses: gorse-cloud/redis-cluster@v1.0.0 | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: Test RISC-V floats | |
| run: make -C common/floats test-rvv | |
| env: | |
| RISCV64_QEMU: qemu-riscv64-static | |
| - name: Test LoongArch LASX floats | |
| run: make -C common/floats test-lasx | |
| env: | |
| LOONG64_QEMU: qemu-loongarch64-static | |
| - name: Test MariaDB | |
| run: go test ./storage/data ./storage/cache -run TestMySQL | |
| env: | |
| MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mariadb.ports[3306] }})/ | |
| - name: Test Kvrocks | |
| run: go test ./storage/cache -run ^TestRedis | |
| env: | |
| REDIS_URI: redis://localhost:${{ job.services.kvrocks.ports[6666] }}/ | |
| - name: Test Redis cluster (1 address) | |
| run: go test ./storage/cache -run ^TestRedis | |
| env: | |
| REDIS_URI: redis+cluster://localhost:7000 | |
| - name: Test Redis cluster (6 addresses) | |
| run: go test ./storage/cache -run ^TestRedis | |
| env: | |
| REDIS_URI: redis+cluster://localhost:7000?addr=localhost:7001&addr=localhost:7002&addr=localhost:7003&addr=localhost:7004&addr=localhost:7005 | |
| playground: | |
| name: playground | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Build Docker image | |
| run: docker build -t zhenghaoz/gorse-in-one -f ./cmd/gorse-in-one/Dockerfile . | |
| - name: Run playground | |
| run: docker run -d -p 8088:8088 --name playground zhenghaoz/gorse-in-one --playground | |
| - name: Check dashboard URL | |
| run: | | |
| for i in {1..10}; do | |
| curl -sSf http://localhost:8088 && break | |
| docker logs playground | |
| sleep 10 | |
| done | |
| golangci: | |
| name: lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: ./go.mod | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v9 | |
| with: | |
| args: --timeout 20m |