MDEV-35897 vector index search allocates too much memory for large ef_search

never estimate that a graph search will visit more nodes than there
are in the graph. In fact, let's reduce the graph size by 30%, it'll
increase the false positive rate of a bloom filter by 2% when
visiting the whole graph, it doesn't affect recall noticeably.

we need to read the shared graph size under a lock. let's store it
in the thread-local unused TABLE::used_stat_records member.

Author: vuvova

sql/vector_mhnsw.cc

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2024, MariaDB plc
+   Copyright (c) 2024, 2025, MariaDB plc
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -753,6 +753,8 @@ MHNSW_Share *MHNSW_Share::get_from_share(TABLE_SHARE *share, TABLE *table)
   }
   if (ctx)
     ctx->refcnt++;
+  if (table) // hijack TABLE::used_stat_records
+    table->hlindex->used_stat_records= ctx->node_cache.size();
   share->unlock_share();
   return ctx;
 }
@@ -1144,8 +1146,9 @@ static int search_layer(MHNSW_Share *ctx, TABLE *graph, const FVector *target,
 
   // WARNING! heuristic here
   const double est_heuristic= 8 * std::sqrt(ctx->max_neighbors(layer));
-  const uint est_size= static_cast<uint>(est_heuristic * std::pow(ef, ctx->ef_power));
-  VisitedSet visited(root, target, est_size);
+  double est_size= est_heuristic * std::pow(ef, ctx->ef_power);
+  set_if_smaller(est_size, graph->used_stat_records/1.3);
+  VisitedSet visited(root, target, static_cast<uint>(est_size));
 
   candidates.init(max_ef, false, Visited::cmp);
   best.init(ef, true, Visited::cmp);
@@ -1213,9 +1216,9 @@ static int search_layer(MHNSW_Share *ctx, TABLE *graph, const FVector *target,
     }
   }
   set_if_bigger(ctx->diameter, max_distance); // not atomic, but it's ok
-  if (ef > 1 && visited.count*2 > est_size)
+  if (ef > 1 && visited.count > est_size)
   {
-    double ef_power= std::log(visited.count*2/est_heuristic) / std::log(ef);
+    double ef_power= std::log(visited.count/est_heuristic) / std::log(ef);
     set_if_bigger(ctx->ef_power, ef_power); // not atomic, but it's ok
   }