Run ci with payload (#231)

* Add with_payload param

* Update poetry

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: tellet-q

.github/workflows/continuous-benchmark.yaml

@@ -3,6 +3,10 @@ name: Continuous Benchmark
 on:
   repository_dispatch:
   workflow_dispatch:
+    inputs:
+      with_payload:
+        description: 'Flag that controls whether to search with or without payload (false or true)'
+        default: false
   schedule:
     # Run every 4 hours
     - cron: "0 */4 * * *"
@@ -27,13 +31,16 @@ jobs:
             export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
             export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
             export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
+            export WITH_PAYLOAD=${{ inputs.with_payload }}
+            export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "")
+
             bash -x tools/setup_ci.sh
 
             declare -A DATASET_TO_ENGINE
-            DATASET_TO_ENGINE["laion-small-clip"]="qdrant-continuous-benchmark"
-            DATASET_TO_ENGINE["msmarco-sparse-100K"]="qdrant-sparse-vector"
-            DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark"
-            DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark"
+            DATASET_TO_ENGINE["laion-small-clip"]="qdrant-continuous-benchmark${SUFFIX}"
+            DATASET_TO_ENGINE["msmarco-sparse-100K"]="qdrant-sparse-vector${SUFFIX}"
+            DATASET_TO_ENGINE["h-and-m-2048-angular-filters"]="qdrant-continuous-benchmark${SUFFIX}"
+            DATASET_TO_ENGINE["dbpedia-openai-100K-1536-angular"]="qdrant-bq-continuous-benchmark${SUFFIX}"
 
             set +e
 
@@ -117,13 +124,16 @@ jobs:
             export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
             export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
             export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
+            export WITH_PAYLOAD=${{ inputs.with_payload }}
+            export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "")
+
             bash -x tools/setup_ci.sh
 
             set +e
 
             # Benchmark filtered search by tenants with mem limitation
 
-            export ENGINE_NAME="qdrant-all-on-disk-scalar-q"
+            export ENGINE_NAME="qdrant-all-on-disk-scalar-q${SUFFIX}"
             export DATASETS="random-768-100-tenants"
             export BENCHMARK_STRATEGY="tenants"
             export CONTAINER_MEM_LIMIT=160mb
@@ -203,13 +213,16 @@ jobs:
             export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
             export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
             export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
+            export WITH_PAYLOAD=${{ inputs.with_payload }}
+            export SUFFIX=$([ "${WITH_PAYLOAD}" = "true" ] && echo "-with-payload" || echo "")
+
             bash -x tools/setup_ci.sh
 
             set +e
 
             # Benchmark parallel search&upload
 
-            export ENGINE_NAME="qdrant-continuous-benchmark"
+            export ENGINE_NAME="qdrant-continuous-benchmark${SUFFIX}"
             export DATASETS="laion-small-clip"
             export BENCHMARK_STRATEGY="parallel"
             export POSTGRES_TABLE="benchmark_parallel_search_upload"

Dockerfile

@@ -6,7 +6,7 @@ ENV PYTHONFAULTHANDLER=1 \
   PIP_NO_CACHE_DIR=off \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
   PIP_DEFAULT_TIMEOUT=100 \
-  POETRY_VERSION=1.5.1
+  POETRY_VERSION=2.1.2
 
 RUN pip install "poetry==$POETRY_VERSION"
 
@@ -16,7 +16,7 @@ COPY poetry.lock pyproject.toml /code/
 
 # Project initialization:
 RUN poetry config virtualenvs.create false \
-    && poetry install --no-dev --no-interaction --no-ansi
+    && poetry --no-interaction --no-ansi install --without dev
 
 # Creating folders, and files for a project:
 COPY . /code

engine/clients/qdrant/search.py

@@ -54,6 +54,7 @@ def search_one(cls, query: Query, top: int) -> List[Tuple[int, float]]:
                 query_filter=cls.parser.parse(query.meta_conditions),
                 limit=top,
                 search_params=rest.SearchParams(**cls.search_params.get("config", {})),
+                with_payload=cls.search_params.get("with_payload", False),
             )
         except Exception as ex:
             print(f"Something went wrong during search: {ex}")

experiments/configurations/qdrant-on-disk.json

@@ -30,5 +30,24 @@
       { "parallel": 8 }
     ],
     "upload_params": { "parallel": 4 }
+  },
+  {
+    "name": "qdrant-all-on-disk-scalar-q-with-payload",
+    "engine": "qdrant",
+    "connection_params": {},
+    "collection_params": {
+      "optimizers_config": { "default_segment_number": 17 },
+      "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": false} },
+      "vectors_config": { "on_disk": true },
+      "hnsw_config": { "on_disk": true, "m": 0, "payload_m": 16 },
+      "on_disk_payload": true,
+      "payload_index_params": {
+        "a": { "is_tenant": true, "on_disk": true }
+      }
+    },
+    "search_params": [
+      { "parallel": 8, "with_payload": true }
+    ],
+    "upload_params": { "parallel": 4 }
   }
 ]

experiments/configurations/qdrant-single-node.json

@@ -45,6 +45,41 @@
     ],
     "upload_params": { "parallel": 16, "batch_size": 1024 }
   },
+  {
+    "name": "qdrant-continuous-benchmark-with-payload",
+    "engine": "qdrant",
+    "connection_params": { "timeout": 30 },
+    "collection_params": {
+      "hnsw_config": {
+        "m": 32,
+        "ef_construct": 256
+      },
+      "quantization_config": {
+        "scalar": {
+          "type": "int8",
+          "quantile": 0.99
+        }
+      },
+      "optimizers_config": {
+        "max_segment_size": 1000000,
+        "default_segment_number": 3,
+        "memmap_threshold": 10000000
+      }
+    },
+    "search_params": [
+      {
+        "parallel": 8,
+        "config": {
+          "hnsw_ef": 256,
+          "quantization": {
+            "oversampling": 2.0
+          }
+        },
+        "with_payload": true
+      }
+    ],
+    "upload_params": { "parallel": 16, "batch_size": 1024 }
+  },
   {
     "name": "qdrant-bq-continuous-benchmark",
     "engine": "qdrant",
@@ -72,6 +107,34 @@
     ],
     "upload_params": { "parallel": 16, "batch_size": 1024 }
   },
+  {
+    "name": "qdrant-bq-continuous-benchmark-with-payload",
+    "engine": "qdrant",
+    "connection_params": { "timeout": 30 },
+    "collection_params": {
+      "hnsw_config": {
+        "m": 32,
+        "ef_construct": 256
+      },
+      "quantization_config": { "binary": {"always_ram": true} },
+      "optimizers_config": {
+        "max_segment_size": 1000000,
+        "default_segment_number": 3,
+        "memmap_threshold": 10000000
+      }
+    },
+    "search_params": [
+      {
+        "parallel": 8,
+        "config": {
+          "hnsw_ef": 256,
+          "quantization": { "rescore": true, "oversampling": 2.0 }
+        },
+        "with_payload": true
+      }
+    ],
+    "upload_params": { "parallel": 16, "batch_size": 1024 }
+  },
   {
     "name": "qdrant-sparse-vector",
     "engine": "qdrant",
@@ -91,6 +154,26 @@
     ],
     "upload_params": { "parallel": 16, "batch_size": 1024 }
   },
+  {
+    "name": "qdrant-sparse-vector-with-payload",
+    "engine": "qdrant",
+    "connection_params": { "timeout": 30 },
+    "collection_params": {
+      "optimizers_config": {
+        "max_segment_size": 1000000,
+        "default_segment_number": 3,
+        "memmap_threshold": 10000000
+      }
+    },
+    "search_params": [
+      {
+        "parallel": 8,
+        "search_params": {},
+        "with_payload": true
+      }
+    ],
+    "upload_params": { "parallel": 16, "batch_size": 1024 }
+  },
   {
     "name": "qdrant-parallel",
     "engine": "qdrant",