Added REDIS_JUST_INDEX config to avoid resending duplicate data on the wire

Author: filipecosta90

engine/clients/redis/config.py

@@ -6,6 +6,7 @@
 REDIS_CLUSTER = bool(int(os.getenv("REDIS_CLUSTER", 0)))
 REDIS_HYBRID_POLICY = os.getenv("REDIS_HYBRID_POLICY", None)
 REDIS_KEEP_DOCUMENTS = bool(os.getenv("REDIS_KEEP_DOCUMENTS", 1))
+REDIS_JUST_INDEX = bool(os.getenv("REDIS_JUST_INDEX", 0))
 GPU_STATS = bool(int(os.getenv("GPU_STATS", 0)))
 GPU_STATS_ENDPOINT = os.getenv("GPU_STATS_ENDPOINT", None)
 

engine/clients/redis/upload.py

@@ -14,7 +14,7 @@
     REDIS_CLUSTER,
     GPU_STATS,
     GPU_STATS_ENDPOINT,
-    REDIS_KEEP_DOCUMENTS,
+    REDIS_JUST_INDEX,
 )
 from engine.clients.redis.helper import convert_to_redis_coords
 
@@ -55,6 +55,8 @@ def init_client(cls, host, distance, connection_params, upload_params):
     def upload_batch(
         cls, ids: List[int], vectors: List[list], metadata: Optional[List[dict]]
     ):
+        if REDIS_JUST_INDEX:
+            return
         for i in range(len(ids)):
             idx = ids[i]
             vector_key = str(idx)

experiments/configurations/create-intel-ce-edition.py

@@ -2,7 +2,7 @@
 
 ms = [4, 8, 16, 32]
 ef_constructs = [4, 8, 16, 32]
-ef_runtimes = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
+ef_runtimes = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768]
 data_type = "FLOAT16"
 for algo in ["hnsw"]:
     configs = []