Merge branch 'main' into add-bwcLucene87Codec

Author: cbuescher

.buildkite/scripts/dra-workflow.sh

@@ -75,6 +75,7 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
 
 echo --- Running release-manager
 
+set +e
 # Artifacts should be generated
 docker run --rm \
   --name release-manager \
@@ -91,4 +92,16 @@ docker run --rm \
   --version "$ES_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
-  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"
+  --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
+2>&1 | tee release-manager.log
+EXIT_CODE=$?
+set -e
+
+# This failure is just generating a ton of noise right now, so let's just ignore it
+# This should be removed once this issue has been fixed
+if grep "elasticsearch-ubi-9.0.0-SNAPSHOT-docker-image.tar.gz" release-manager.log; then
+  echo "Ignoring error about missing ubi artifact"
+  exit 0
+fi
+
+exit "$EXIT_CODE"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:973431347ad45f40e01afbbd010bf9de929c088a63382239b90dd84f39618bc8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
         "-wolfi",
         "apk"
     ),

docs/changelog/117404.yaml

@@ -0,0 +1,5 @@
+pr: 117404
+summary: Correct bit * byte and bit * float script comparisons
+area: Vector Search
+type: bug
+issues: []

docs/reference/vectors/vector-functions.asciidoc

@@ -336,6 +336,10 @@ When using `bit` vectors, not all the vector functions are available. The suppor
 this is the sum of the bitwise AND of the two vectors. If providing `float[]` or `byte[]`, who has `dims` number of elements, as a query vector, the `dotProduct` is
 the sum of the floating point values using the stored `bit` vector as a mask.
 
+NOTE: When comparing `floats` and `bytes` with `bit` vectors, the `bit` vector is treated as a mask in big-endian order.
+For example, if the `bit` vector is `10100001` (e.g. the single byte value `161`) and its compared
+with array of values `[1, 2, 3, 4, 5, 6, 7, 8]` the `dotProduct` will be `1 + 3 + 8 = 16`.
+
 Here is an example of using dot-product with bit vectors.
 
 [source,console]

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java

@@ -51,6 +51,8 @@ public static long ipByteBinByte(byte[] q, byte[] d) {
     /**
      * Compute the inner product of two vectors, where the query vector is a byte vector and the document vector is a bit vector.
      * This will return the sum of the query vector values using the document vector as a mask.
+     * When comparing the bits with the bytes, they are done in "big endian" order. For example, if the byte vector
+     * is [1, 2, 3, 4, 5, 6, 7, 8] and the bit vector is [0b10000000], the inner product will be 1.0.
      * @param q the query vector
      * @param d the document vector
      * @return the inner product of the two vectors
@@ -63,9 +65,9 @@ public static int ipByteBit(byte[] q, byte[] d) {
         // now combine the two vectors, summing the byte dimensions where the bit in d is `1`
         for (int i = 0; i < d.length; i++) {
             byte mask = d[i];
-            for (int j = 0; j < Byte.SIZE; j++) {
+            for (int j = Byte.SIZE - 1; j >= 0; j--) {
                 if ((mask & (1 << j)) != 0) {
-                    result += q[i * Byte.SIZE + j];
+                    result += q[i * Byte.SIZE + Byte.SIZE - 1 - j];
                 }
             }
         }
@@ -75,6 +77,8 @@ public static int ipByteBit(byte[] q, byte[] d) {
     /**
      * Compute the inner product of two vectors, where the query vector is a float vector and the document vector is a bit vector.
      * This will return the sum of the query vector values using the document vector as a mask.
+     * When comparing the bits with the floats, they are done in "big endian" order. For example, if the float vector
+     * is [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] and the bit vector is [0b10000000], the inner product will be 1.0.
      * @param q the query vector
      * @param d the document vector
      * @return the inner product of the two vectors
@@ -86,9 +90,9 @@ public static float ipFloatBit(float[] q, byte[] d) {
         float result = 0;
         for (int i = 0; i < d.length; i++) {
             byte mask = d[i];
-            for (int j = 0; j < Byte.SIZE; j++) {
+            for (int j = Byte.SIZE - 1; j >= 0; j--) {
                 if ((mask & (1 << j)) != 0) {
-                    result += q[i * Byte.SIZE + j];
+                    result += q[i * Byte.SIZE + Byte.SIZE - 1 - j];
                 }
             }
         }

libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java

@@ -21,6 +21,22 @@ public class ESVectorUtilTests extends BaseVectorizationTests {
     static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider();
     static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider();
 
+    public void testIpByteBit() {
+        byte[] q = new byte[16];
+        byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) };
+        random().nextBytes(q);
+        int expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15];
+        assertEquals(expected, ESVectorUtil.ipByteBit(q, d));
+    }
+
+    public void testIpFloatBit() {
+        float[] q = new float[16];
+        byte[] d = new byte[] { (byte) Integer.parseInt("01100010", 2), (byte) Integer.parseInt("10100111", 2) };
+        random().nextFloat();
+        float expected = q[1] + q[2] + q[6] + q[8] + q[10] + q[13] + q[14] + q[15];
+        assertEquals(expected, ESVectorUtil.ipFloatBit(q, d), 1e-6);
+    }
+
     public void testBitAndCount() {
         testBasicBitAndImpl(ESVectorUtil::andBitCountLong);
     }

modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java

@@ -48,7 +48,8 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
         checker.registerConflictCheck(
             "enabled",
             timestampMapping(true, b -> b.startObject("@timestamp").field("type", "date").endObject()),
-            timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject())
+            timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),
+            dm -> {}
         );
         checker.registerUpdateCheck(
             timestampMapping(false, b -> b.startObject("@timestamp").field("type", "date").endObject()),

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/141_multi_dense_vector_max_sim.yml

@@ -3,7 +3,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ multi_dense_vector_script_max_sim ]
+          capabilities: [ multi_dense_vector_script_max_sim_with_bugfix ]
       test_runner_features: capabilities
       reason: "Support for multi dense vector max-sim functions capability required"
   - skip:
@@ -136,10 +136,10 @@ setup:
   - match: {hits.total: 2}
 
   - match: {hits.hits.0._id: "1"}
-  - close_to: {hits.hits.0._score: {value: 190, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 220, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score: {value: 125, error: 0.01}}
+  - close_to: {hits.hits.1._score: {value: 147, error: 0.01}}
 ---
 "Test max-sim inv hamming scoring":
   - skip:

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml

@@ -108,7 +108,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       reason: Capability required to run test
   - do:
       catch: bad_request
@@ -399,7 +399,7 @@ setup:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       test_runner_features: [capabilities, close_to]
       reason: Capability required to run test
   - do:
@@ -419,13 +419,13 @@ setup:
   - match: { hits.total: 3 }
 
   - match: {hits.hits.0._id: "2"}
-  - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}}
+  - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}}
 
   - match: {hits.hits.2._id: "1"}
-  - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}}
+  - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}}
 
   - do:
       headers:
@@ -444,20 +444,20 @@ setup:
   - match: { hits.total: 3 }
 
   - match: {hits.hits.0._id: "2"}
-  - close_to: {hits.hits.0._score: {value: 35.999, error: 0.01}}
+  - close_to: {hits.hits.0._score: {value: 33.78, error: 0.01}}
 
   - match: {hits.hits.1._id: "3"}
-  - close_to: {hits.hits.1._score:{value: 27.23, error: 0.01}}
+  - close_to: {hits.hits.1._score:{value: 22.579, error: 0.01}}
 
   - match: {hits.hits.2._id: "1"}
-  - close_to: {hits.hits.2._score: {value: 16.57, error: 0.01}}
+  - close_to: {hits.hits.2._score: {value: 11.919, error: 0.01}}
 ---
 "Dot product with byte":
   - requires:
       capabilities:
         - method: POST
           path: /_search
-          capabilities: [ byte_float_bit_dot_product ]
+          capabilities: [ byte_float_bit_dot_product_with_bugfix ]
       test_runner_features: capabilities
       reason: Capability required to run test
   - do:
@@ -476,14 +476,14 @@ setup:
 
   - match: { hits.total: 3 }
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0._score: 248}
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0._score: 415}
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1._score: 136}
+  - match: {hits.hits.1._id: "1"}
+  - match: {hits.hits.1._score: 168}
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2._score: 20}
+  - match: {hits.hits.2._id: "2"}
+  - match: {hits.hits.2._score: 126}
 
   - do:
       headers:
@@ -501,11 +501,11 @@ setup:
 
   - match: { hits.total: 3 }
 
-  - match: {hits.hits.0._id: "1"}
-  - match: {hits.hits.0._score: 248}
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0._score: 415}
 
-  - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.1._score: 136}
+  - match: {hits.hits.1._id: "1"}
+  - match: {hits.hits.1._score: 168}
 
-  - match: {hits.hits.2._id: "3"}
-  - match: {hits.hits.2._score: 20}
+  - match: {hits.hits.2._id: "2"}
+  - match: {hits.hits.2._score: 126}

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java

@@ -33,6 +33,7 @@
 import io.netty.handler.timeout.ReadTimeoutException;
 import io.netty.handler.timeout.ReadTimeoutHandler;
 import io.netty.util.AttributeKey;
+import io.netty.util.ResourceLeakDetector;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -410,6 +411,9 @@ protected Result beginEncode(HttpResponse httpResponse, String acceptEncoding) t
                     }
                 });
             }
+            if (ResourceLeakDetector.isEnabled()) {
+                ch.pipeline().addLast(new Netty4LeakDetectionHandler());
+            }
             ch.pipeline()
                 .addLast(
                     "pipelining",

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4LeakDetectionHandler.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.http.netty4;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpRequest;
+
+import org.elasticsearch.tasks.Task;
+
+/**
+ * Inbound channel handler that enrich leaking buffers information from HTTP request.
+ * It helps to detect which handler is leaking buffers. Especially integration tests that run with
+ * paranoid leak detector that samples all buffers for leaking. Supplying informative opaque-id in
+ * integ test helps to narrow down problem (for example test name).
+ */
+public class Netty4LeakDetectionHandler extends ChannelInboundHandlerAdapter {
+
+    private String info;
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg) {
+        if (msg instanceof HttpRequest request) {
+            var opaqueId = request.headers().get(Task.X_OPAQUE_ID_HTTP_HEADER);
+            info = "method: " + request.method() + "; uri: " + request.uri() + "; x-opaque-id: " + opaqueId;
+        }
+        if (msg instanceof HttpContent content) {
+            content.touch(info);
+        }
+        ctx.fireChannelRead(msg);
+    }
+}

muted-tests.yml

@@ -185,9 +185,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled}
   issue: https://github.com/elastic/elasticsearch/issues/116775
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
-  issue: https://github.com/elastic/elasticsearch/issues/116777
 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/116851
 - class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT
@@ -235,6 +232,12 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473
+- class: org.elasticsearch.xpack.esql.qa.single_node.FieldExtractorIT
+  method: testConstantKeywordField
+  issue: https://github.com/elastic/elasticsearch/issues/117524
+- class: org.elasticsearch.xpack.esql.qa.multi_node.FieldExtractorIT
+  method: testConstantKeywordField
+  issue: https://github.com/elastic/elasticsearch/issues/117524
 
 # Examples:
 #

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -417,9 +418,15 @@ public void testSyntheticSource() throws IOException {
         if (isOldCluster()) {
             Request createIndex = new Request("PUT", "/synthetic");
             XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
+            boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion());
+            if (useIndexSetting) {
+                indexSpec.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject();
+            }
             indexSpec.startObject("mappings");
             {
-                indexSpec.startObject("_source").field("mode", "synthetic").endObject();
+                if (useIndexSetting == false) {
+                    indexSpec.startObject("_source").field("mode", "synthetic").endObject();
+                }
                 indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject();
             }
             indexSpec.endObject();

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/10_synonyms_put.yml

@@ -17,7 +17,10 @@ setup:
 
   - do:
       cluster.health:
-        wait_for_no_initializing_shards: true
+        index: .synonyms-2
+        timeout: 2s
+        wait_for_status: green
+        ignore: 408
 
   - do:
       synonyms.get_synonym:
@@ -64,7 +67,10 @@ setup:
 
   - do:
       cluster.health:
-        wait_for_no_initializing_shards: true
+        index: .synonyms-2
+        timeout: 2s
+        wait_for_status: green
+        ignore: 408
 
   - do:
       synonyms.get_synonym:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/110_synonyms_invalid.yml

@@ -14,7 +14,10 @@ setup:
   # This is to ensure that all index shards (write and read) are available. In serverless this can take some time.
   - do:
       cluster.health:
-        wait_for_no_initializing_shards: true
+        index: .synonyms-2
+        timeout: 2s
+        wait_for_status: green
+        ignore: 408
 
   - do:
       indices.create: