|
11 | 11 |
|
12 | 12 | import org.apache.http.client.fluent.Request; |
13 | 13 | import org.apache.http.entity.ContentType; |
| 14 | +import org.elasticsearch.packaging.util.FileUtils; |
| 15 | +import org.elasticsearch.packaging.util.Platforms; |
14 | 16 | import org.elasticsearch.packaging.util.ServerUtils; |
| 17 | +import org.elasticsearch.packaging.util.docker.Docker; |
15 | 18 | import org.elasticsearch.packaging.util.docker.DockerRun; |
16 | 19 |
|
17 | 20 | import java.util.Map; |
| 21 | +import java.util.regex.Matcher; |
| 22 | +import java.util.regex.Pattern; |
18 | 23 |
|
19 | 24 | import static org.elasticsearch.packaging.util.docker.Docker.runContainer; |
20 | 25 | import static org.elasticsearch.packaging.util.docker.DockerRun.builder; |
@@ -79,6 +84,99 @@ public void test20ZstdCompression() throws Exception { |
79 | 84 | } |
80 | 85 | } |
81 | 86 |
|
| 87 | + /** |
| 88 | + * Verifies that the native simdvec library (libvec.so/libvec.dylib) loads and can perform vector similarity |
| 89 | + * scoring. |
| 90 | + * <p> |
| 91 | + * A {@code dense_vector} field with HNSW indexing uses the native vector scorer from {@code libvec} |
| 92 | + * for computing vector distances during kNN search. If the native library cannot be loaded (e.g. due to |
| 93 | + * a glibc version incompatibility), the search will fall back to the Java implementation or fail entirely. |
| 94 | + * This test indexes vectors and performs a kNN search to exercise the native scoring path. |
| 95 | + * <p> |
| 96 | + * On Linux and macOS (where native simdvec is supported), this test additionally asserts that the native |
| 97 | + * library was loaded successfully by checking for the {@code vec_caps=N} log line (where N > 0) emitted |
| 98 | + * by the simdvec library during startup. |
| 99 | + */ |
| 100 | + public void test30SimdVecKnnSearch() throws Exception { |
| 101 | + configureAndStart(SECURITY_DISABLED_SETTINGS); |
| 102 | + |
| 103 | + try { |
| 104 | + // Create an index with a dense_vector field using plain HNSW (no quantization). |
| 105 | + // Explicitly setting "type": "hnsw" avoids the default int8_hnsw quantization, |
| 106 | + // ensuring the native float32 vector scorer in libvec is used for distance computation. |
| 107 | + ServerUtils.makeRequest(Request.Put("http://localhost:9200/simdvec_test").bodyString(""" |
| 108 | + { |
| 109 | + "settings": {"number_of_replicas": 0, "number_of_shards": 1}, |
| 110 | + "mappings": { |
| 111 | + "properties": { |
| 112 | + "vector": { |
| 113 | + "type": "dense_vector", |
| 114 | + "dims": 3, |
| 115 | + "index": true, |
| 116 | + "similarity": "l2_norm", |
| 117 | + "index_options": {"type": "hnsw"} |
| 118 | + }, |
| 119 | + "name": {"type": "keyword"} |
| 120 | + } |
| 121 | + } |
| 122 | + }""", ContentType.APPLICATION_JSON)); |
| 123 | + |
| 124 | + // Index documents with vectors |
| 125 | + ServerUtils.makeRequest( |
| 126 | + Request.Post("http://localhost:9200/simdvec_test/_doc/1") |
| 127 | + .bodyString("{\"vector\": [1.0, 2.0, 3.0], \"name\": \"first\"}", ContentType.APPLICATION_JSON) |
| 128 | + ); |
| 129 | + ServerUtils.makeRequest( |
| 130 | + Request.Post("http://localhost:9200/simdvec_test/_doc/2") |
| 131 | + .bodyString("{\"vector\": [4.0, 5.0, 6.0], \"name\": \"second\"}", ContentType.APPLICATION_JSON) |
| 132 | + ); |
| 133 | + ServerUtils.makeRequest( |
| 134 | + Request.Post("http://localhost:9200/simdvec_test/_doc/3?refresh=true") |
| 135 | + .bodyString("{\"vector\": [7.0, 8.0, 9.0], \"name\": \"third\"}", ContentType.APPLICATION_JSON) |
| 136 | + ); |
| 137 | + |
| 138 | + // Force merge to a single segment to ensure the HNSW graph is built and the native scorer is used |
| 139 | + ServerUtils.makeRequest(Request.Post("http://localhost:9200/simdvec_test/_forcemerge?max_num_segments=1")); |
| 140 | + |
| 141 | + // Perform a kNN search — this exercises native vector distance scoring via libvec |
| 142 | + String response = ServerUtils.makeRequest(Request.Post("http://localhost:9200/simdvec_test/_search").bodyString(""" |
| 143 | + { |
| 144 | + "knn": { |
| 145 | + "field": "vector", |
| 146 | + "query_vector": [1.0, 2.0, 3.0], |
| 147 | + "k": 1, |
| 148 | + "num_candidates": 3 |
| 149 | + } |
| 150 | + }""", ContentType.APPLICATION_JSON)); |
| 151 | + |
| 152 | + // The nearest neighbor to [1,2,3] should be the document with vector [1,2,3] |
| 153 | + assertThat(response, containsString("\"_id\":\"1\"")); |
| 154 | + assertThat(response, containsString("\"first\"")); |
| 155 | + |
| 156 | + // On Linux and macOS (where native simdvec is supported), verify that libvec loaded and vec_caps > 0. |
| 157 | + // The vec_caps log line is emitted during NativeAccess initialization at node startup. |
| 158 | + if (Platforms.LINUX || Platforms.DARWIN) { |
| 159 | + String logs = getElasticsearchLogs(); |
| 160 | + Matcher matcher = Pattern.compile("vec_caps=(\\d+)").matcher(logs); |
| 161 | + assertTrue("Expected vec_caps=N log line indicating simdvec library loaded, but not found in logs", matcher.find()); |
| 162 | + int vecCaps = Integer.parseInt(matcher.group(1)); |
| 163 | + assertTrue("Expected vec_caps > 0, indicating native simdvec is operational, but got: " + vecCaps, vecCaps > 0); |
| 164 | + } |
| 165 | + } finally { |
| 166 | + stopElasticsearch(); |
| 167 | + } |
| 168 | + } |
| 169 | + |
| 170 | + /** |
| 171 | + * Returns the Elasticsearch startup logs, handling both Docker and non-Docker distributions. |
| 172 | + */ |
| 173 | + private String getElasticsearchLogs() { |
| 174 | + if (distribution().isDocker()) { |
| 175 | + return Docker.getContainerLogs().stdout(); |
| 176 | + } |
| 177 | + return FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"); |
| 178 | + } |
| 179 | + |
82 | 180 | private void configureAndStart(Map<String, String> settings) throws Exception { |
83 | 181 | if (distribution().isDocker()) { |
84 | 182 | DockerRun dockerRun = builder(); |
|
0 commit comments