Merge branch 'main' into add-bwcLucene87Codec

Author: cbuescher

.editorconfig

@@ -209,7 +209,7 @@ indent_size = 4
 max_line_length = 140
 ij_java_class_count_to_use_import_on_demand = 999
 ij_java_names_count_to_use_import_on_demand = 999
-ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,javax.**,|,$*
+ij_java_imports_layout = *,|,com.**,|,org.**,|,java.**,|,javax.**,|,$*
 
 [*.json]
 indent_size = 2

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -142,13 +142,18 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    doLast {
-      ['main', 'test'].each { sourceSet ->
-        modifyXml(".idea/modules/libs/native/elasticsearch.libs.native.${sourceSet}.iml") { xml ->
-          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = 'JDK_21_PREVIEW'
+    ext {
+      enablePreview = { moduleFile, languageLevel ->
+        modifyXml(moduleFile) { xml ->
+          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
     }
+
+    doLast {
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
+      enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
+    }
   }
 
   tasks.register('buildDependencyArtifacts') {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:55b297da5151d2a2997e8ab9729fe1304e4869389d7090ab7031cc29530f69f8",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -167,5 +167,3 @@ org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.Str
 @defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
 java.lang.Thread#<init>(java.lang.Runnable)
 java.lang.Thread#<init>(java.lang.ThreadGroup, java.lang.Runnable)
-
-org.elasticsearch.common.bytes.BytesReference#copyBytes(org.elasticsearch.common.bytes.BytesReference) @ This method is a subject for removal. Copying bytes is prone to performance regressions and unnecessary allocations.

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -98,7 +98,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final int ES_DESTROY_TIMEOUT = 20;
     private static final TimeUnit ES_DESTROY_TIMEOUT_UNIT = TimeUnit.SECONDS;
 
-    private static final int NODE_UP_TIMEOUT = 2;
+    private static final int NODE_UP_TIMEOUT = 3;
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES;
     private static final int ADDITIONAL_CONFIG_TIMEOUT = 15;
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;

docs/changelog/116739.yaml

@@ -0,0 +1,5 @@
+pr: 116739
+summary: Change default Docker image to be based on UBI minimal instead of Ubuntu
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/117235.yaml

@@ -0,0 +1,5 @@
+pr: 117235
+summary: "Deprecate `ChunkingOptions` parameter"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/117303.yaml

@@ -0,0 +1,5 @@
+pr: 117303
+summary: Remove HTTP content copies
+area: Network
+type: enhancement
+issues: []

docs/changelog/117503.yaml

@@ -0,0 +1,6 @@
+pr: 117503
+summary: Fix COUNT filter pushdown
+area: ES|QL
+type: bug
+issues:
+ - 115522

docs/changelog/117551.yaml

@@ -0,0 +1,5 @@
+pr: 117551
+summary: Fix stats by constant expresson with alias
+area: ES|QL
+type: bug
+issues: []

docs/changelog/117595.yaml

@@ -0,0 +1,5 @@
+pr: 117595
+summary: Fix for Deberta tokenizer when input sequence exceeds 512 tokens
+area: Machine Learning
+type: bug
+issues: []

docs/reference/connector/docs/connectors-content-syncs.asciidoc

@@ -52,7 +52,7 @@ However, a fast, accessible third-party data source that stores huge amounts of
 
 [NOTE]
 ====
-Incremental syncs for the SharePoint Online connector use specific logic.
+Incremental syncs for <<es-connectors-sharepoint-online,SharePoint Online>> and <<es-connectors-google-drive,Google Drive>> connectors use specific logic.
 All other connectors use the same shared connector framework logic for incremental syncs.
 ====
 

docs/reference/data-streams/tsds.asciidoc

@@ -339,4 +339,5 @@ include::tsds-index-settings.asciidoc[]
 include::downsampling.asciidoc[]
 include::downsampling-ilm.asciidoc[]
 include::downsampling-manual.asciidoc[]
+include::downsampling-dsl.asciidoc[]
 include::tsds-reindex.asciidoc[]

docs/reference/inference/service-elser.asciidoc

@@ -102,10 +102,39 @@ If `adaptive_allocations` is enabled, do not set this value, because it's automa
 Sets the number of threads used by each model allocation during inference. This generally increases the speed per inference request. The inference process is a compute-bound process; `threads_per_allocations` must not exceed the number of available allocated processors per node.
 Must be a power of 2. Max allowed value is 32.
 
+[discrete]
+[[inference-example-elser-adaptive-allocation]]
+==== ELSER service example with adaptive allocations
+
+When adaptive allocations are enabled, the number of allocations of the model is set automatically based on the current load.
+
+NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
+
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
+
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/sparse_embedding/my-elser-model
+{
+  "service": "elser",
+  "service_settings": {
+    "adaptive_allocations": {
+      "enabled": true,
+      "min_number_of_allocations": 3,
+      "max_number_of_allocations": 10
+    },
+    "num_threads": 1
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
 
 [discrete]
 [[inference-example-elser]]
-==== ELSER service example
+==== ELSER service example without adaptive allocations
 
 The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
@@ -151,32 +180,4 @@ You might see a 502 bad gateway error in the response when using the {kib} Conso
 This error usually just reflects a timeout, while the model downloads in the background.
 You can check the download progress in the {ml-app} UI.
 If using the Python client, you can set the `timeout` parameter to a higher value.
-====
-
-[discrete]
-[[inference-example-elser-adaptive-allocation]]
-==== Setting adaptive allocations for the ELSER service
-
-NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
-To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
-
-The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
-
-The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/sparse_embedding/my-elser-model
-{
-  "service": "elser",
-  "service_settings": {
-    "adaptive_allocations": {
-      "enabled": true,
-      "min_number_of_allocations": 3,
-      "max_number_of_allocations": 10
-    },
-    "num_threads": 1
-  }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
+====

docs/reference/intro.asciidoc

@@ -85,7 +85,7 @@ You can deploy {es} in various ways.
 **Hosted options**
 
 * {cloud}/ec-getting-started-trial.html[*Elastic Cloud Hosted*]: {es} is available as part of the hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14-day free trial].
-* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
+* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless*]: Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14-day free trial].
 
 **Advanced options**
 

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java

@@ -91,15 +91,18 @@ static MethodKey parseCheckerMethodSignature(String checkerMethodName, Type[] ch
                 String.format(
                     Locale.ROOT,
                     "Checker method %s has incorrect name format. "
-                        + "It should be either check$$methodName (instance) or check$package_ClassName$methodName (static)",
+                        + "It should be either check$$methodName (instance), check$package_ClassName$methodName (static) or "
+                        + "check$package_ClassName$ (ctor)",
                     checkerMethodName
                 )
             );
         }
 
-        // No "className" (check$$methodName) -> method is static, and we'll get the class from the actual typed argument
+        // No "className" (check$$methodName) -> method is instance, and we'll get the class from the actual typed argument
         final boolean targetMethodIsStatic = classNameStartIndex + 1 != classNameEndIndex;
-        final String targetMethodName = checkerMethodName.substring(classNameEndIndex + 1);
+        // No "methodName" (check$package_ClassName$) -> method is ctor
+        final boolean targetMethodIsCtor = classNameEndIndex + 1 == checkerMethodName.length();
+        final String targetMethodName = targetMethodIsCtor ? "<init>" : checkerMethodName.substring(classNameEndIndex + 1);
 
         final String targetClassName;
         final List<String> targetParameterTypes;

libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java

@@ -154,11 +154,12 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, Str
             var mv = super.visitMethod(access, name, descriptor, signature, exceptions);
             if (isAnnotationPresent == false) {
                 boolean isStatic = (access & ACC_STATIC) != 0;
+                boolean isCtor = "<init>".equals(name);
                 var key = new MethodKey(className, name, Stream.of(Type.getArgumentTypes(descriptor)).map(Type::getInternalName).toList());
                 var instrumentationMethod = instrumentationMethods.get(key);
                 if (instrumentationMethod != null) {
                     // LOGGER.debug("Will instrument method {}", key);
-                    return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, descriptor, instrumentationMethod);
+                    return new EntitlementMethodVisitor(Opcodes.ASM9, mv, isStatic, isCtor, descriptor, instrumentationMethod);
                 } else {
                     // LOGGER.trace("Will not instrument method {}", key);
                 }
@@ -187,6 +188,7 @@ private void addClassAnnotationIfNeeded() {
 
     class EntitlementMethodVisitor extends MethodVisitor {
         private final boolean instrumentedMethodIsStatic;
+        private final boolean instrumentedMethodIsCtor;
         private final String instrumentedMethodDescriptor;
         private final CheckerMethod instrumentationMethod;
         private boolean hasCallerSensitiveAnnotation = false;
@@ -195,11 +197,13 @@ class EntitlementMethodVisitor extends MethodVisitor {
             int api,
             MethodVisitor methodVisitor,
             boolean instrumentedMethodIsStatic,
+            boolean instrumentedMethodIsCtor,
             String instrumentedMethodDescriptor,
             CheckerMethod instrumentationMethod
         ) {
             super(api, methodVisitor);
             this.instrumentedMethodIsStatic = instrumentedMethodIsStatic;
+            this.instrumentedMethodIsCtor = instrumentedMethodIsCtor;
             this.instrumentedMethodDescriptor = instrumentedMethodDescriptor;
             this.instrumentationMethod = instrumentationMethod;
         }
@@ -260,14 +264,15 @@ private void pushCallerClass() {
 
         private void forwardIncomingArguments() {
             int localVarIndex = 0;
-            if (instrumentedMethodIsStatic == false) {
+            if (instrumentedMethodIsCtor) {
+                localVarIndex++;
+            } else if (instrumentedMethodIsStatic == false) {
                 mv.visitVarInsn(Opcodes.ALOAD, localVarIndex++);
             }
             for (Type type : Type.getArgumentTypes(instrumentedMethodDescriptor)) {
                 mv.visitVarInsn(type.getOpcode(Opcodes.ILOAD), localVarIndex);
                 localVarIndex += type.getSize();
             }
-
         }
 
         private void invokeInstrumentationMethod() {

libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java

@@ -45,6 +45,12 @@ interface TestCheckerOverloads {
         void check$org_example_TestTargetClass$staticMethodWithOverload(Class<?> clazz, int x, String y);
     }
 
+    interface TestCheckerCtors {
+        void check$org_example_TestTargetClass$(Class<?> clazz);
+
+        void check$org_example_TestTargetClass$(Class<?> clazz, int x, String y);
+    }
+
     public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundException {
         Map<MethodKey, CheckerMethod> methodsMap = instrumentationService.lookupMethodsToInstrument(TestChecker.class.getName());
 
@@ -142,6 +148,38 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException, C
         );
     }
 
+    public void testInstrumentationTargetLookupWithCtors() throws IOException, ClassNotFoundException {
+        Map<MethodKey, CheckerMethod> methodsMap = instrumentationService.lookupMethodsToInstrument(TestCheckerCtors.class.getName());
+
+        assertThat(methodsMap, aMapWithSize(2));
+        assertThat(
+            methodsMap,
+            hasEntry(
+                equalTo(new MethodKey("org/example/TestTargetClass", "<init>", List.of("I", "java/lang/String"))),
+                equalTo(
+                    new CheckerMethod(
+                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors",
+                        "check$org_example_TestTargetClass$",
+                        List.of("Ljava/lang/Class;", "I", "Ljava/lang/String;")
+                    )
+                )
+            )
+        );
+        assertThat(
+            methodsMap,
+            hasEntry(
+                equalTo(new MethodKey("org/example/TestTargetClass", "<init>", List.of())),
+                equalTo(
+                    new CheckerMethod(
+                        "org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests$TestCheckerCtors",
+                        "check$org_example_TestTargetClass$",
+                        List.of("Ljava/lang/Class;")
+                    )
+                )
+            )
+        );
+    }
+
     public void testParseCheckerMethodSignatureStaticMethod() {
         var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
             "check$org_example_TestClass$staticMethod",
@@ -169,6 +207,24 @@ public void testParseCheckerMethodSignatureStaticMethodInnerClass() {
         assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass$InnerClass", "staticMethod", List.of())));
     }
 
+    public void testParseCheckerMethodSignatureCtor() {
+        var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
+            "check$org_example_TestClass$",
+            new Type[] { Type.getType(Class.class) }
+        );
+
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "<init>", List.of())));
+    }
+
+    public void testParseCheckerMethodSignatureCtorWithArgs() {
+        var methodKey = InstrumentationServiceImpl.parseCheckerMethodSignature(
+            "check$org_example_TestClass$",
+            new Type[] { Type.getType(Class.class), Type.getType("I"), Type.getType(String.class) }
+        );
+
+        assertThat(methodKey, equalTo(new MethodKey("org/example/TestClass", "<init>", List.of("I", "java/lang/String"))));
+    }
+
     public void testParseCheckerMethodSignatureIncorrectName() {
         var exception = assertThrows(
             IllegalArgumentException.class,