diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d10cbc65426f0..91c9fd1cc6c76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -698,6 +698,27 @@ If your changes affect only the documentation, run: For more information about testing code examples in the documentation, see https://github.com/elastic/elasticsearch/blob/master/docs/README.asciidoc +### Only running failed tests + +When you open your pull-request it may be approved for review. If so, the full +test suite is run within Elasticsearch's CI environment. If a test fails, +you can see how to run that particular test by searching for the `REPRODUCE` +string in the CI's console output. + +Elasticsearch's testing suite takes advantage of randomized testing. Consequently, +a test that passes locally, may actually fail later due to random settings +or data input. To make tests repeatable, a `REPRODUCE` line in CI will also include +the `-Dtests.seed` parameter. + +When running locally, gradle does its best to take advantage of cached results. +So, if the code is unchanged, running the same test with the same `-Dtests.seed` +repeatedly may not actually run the test if it has passed with that seed + in the previous execution. A way around this is to pass a separate parameter +to adjust the command options seen by gradle. +A simple option may be to add the parameter `-Dtests.timestamp=$(date +%s)` +which will give the current time stamp as a parameter, thus making the parameters +sent to gradle unique and bypassing the cache. + ### Project layout This repository is split into many top level directories. The most important diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy index e39d178bff2b6..99be5d62e5d97 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy @@ -17,8 +17,20 @@ class LocalRepositoryFixture extends ExternalResource{ private TemporaryFolder temporaryFolder - LocalRepositoryFixture(TemporaryFolder temporaryFolder){ - this.temporaryFolder = temporaryFolder + LocalRepositoryFixture(){ + this.temporaryFolder = new TemporaryFolder() + } + + @Override + protected void before() throws Throwable { + super.before() + temporaryFolder.before() + } + + @Override + protected void after() { + super.after() + temporaryFolder.after() } void generateJar(String group, String module, String version, String... clazzNames){ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy index b950fadecce78..1c64f4c9bf5d5 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy @@ -10,7 +10,11 @@ package org.elasticsearch.gradle.internal import org.apache.commons.io.IOUtils import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture import org.gradle.testkit.runner.TaskOutcome +import org.junit.ClassRule +import org.junit.rules.TemporaryFolder +import spock.lang.Shared import java.nio.charset.StandardCharsets import java.util.zip.ZipEntry @@ -20,6 +24,10 @@ import static org.elasticsearch.gradle.fixtures.TestClasspathUtils.setupJarHellJ class BuildPluginFuncTest extends AbstractGradleFuncTest { + @Shared + @ClassRule + public LocalRepositoryFixture repository = new LocalRepositoryFixture() + def EXAMPLE_LICENSE = """\ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -120,6 +128,8 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest { def "applies checks"() { given: + repository.generateJar("org.elasticsearch", "build-conventions", "unspecified", 'org.acme.CheckstyleStuff') + repository.configureBuild(buildFile) setupJarHellJar(dir('local-repo/org/elasticsearch/elasticsearch-core/current/')) file("licenses/hamcrest-core-1.3.jar.sha1").text = "42a25dc3219429f0e5d060061f71acb49bf010a0" file("licenses/hamcrest-core-LICENSE.txt").text = EXAMPLE_LICENSE diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy index 5fd7aedecb268..ff329e766bfe7 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy @@ -8,24 +8,22 @@ package org.elasticsearch.gradle.internal -import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest +import org.gradle.api.Plugin -class ElasticsearchJavaPluginFuncTest extends AbstractGradleFuncTest { +class ElasticsearchJavaPluginFuncTest extends AbstractGradleInternalPluginFuncTest { + + Class pluginClassUnderTest = ElasticsearchJavaPlugin.class def "compatibility options are resolved from from build params minimum runtime version"() { when: - buildFile.text = """ - plugins { - id 'elasticsearch.global-build-info' - } + buildFile.text << """ import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.internal.info.BuildParams BuildParams.init { it.setMinimumRuntimeVersion(JavaVersion.VERSION_1_10) } - apply plugin:'elasticsearch.java' - - assert compileJava.sourceCompatibility == JavaVersion.VERSION_1_10.toString() - assert compileJava.targetCompatibility == JavaVersion.VERSION_1_10.toString() + assert tasks.named('compileJava').get().sourceCompatibility == JavaVersion.VERSION_1_10.toString() + assert tasks.named('compileJava').get().targetCompatibility == JavaVersion.VERSION_1_10.toString() """ then: @@ -34,14 +32,10 @@ class ElasticsearchJavaPluginFuncTest extends AbstractGradleFuncTest { def "compile option --release is configured from targetCompatibility"() { when: - buildFile.text = """ - plugins { - id 'elasticsearch.java' - } - - compileJava.targetCompatibility = "1.10" + buildFile.text << """ + tasks.named('compileJava').get().targetCompatibility = "1.10" afterEvaluate { - assert compileJava.options.release.get() == 10 + assert tasks.named('compileJava').get().options.release.get() == 10 } """ then: diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy index 756562ab02725..8a10ad587cb8b 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy @@ -14,6 +14,8 @@ import org.gradle.testkit.runner.TaskOutcome class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { + // using LoggedExec is not cc compatible + configurationCacheCompatible = false internalBuild() buildFile << """ import org.elasticsearch.gradle.Version; diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index 2d1a6193189d7..48b2b52820c8a 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -22,6 +22,8 @@ import spock.lang.Unroll class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { + // used LoggedExec task is not configuration cache compatible and + configurationCacheCompatible = false internalBuild() buildFile << """ apply plugin: 'elasticsearch.internal-distribution-bwc-setup' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy index 63aa65fec359e..8300318fbdc16 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy @@ -147,6 +147,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.jdk-download' } + import org.elasticsearch.gradle.internal.Jdk apply plugin: 'base' apply plugin: 'elasticsearch.jdk-download' @@ -158,11 +159,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { architecture = "x64" } } - - tasks.register("getJdk") { + + tasks.register("getJdk", PrintJdk) { dependsOn jdks.myJdk - doLast { - println "JDK HOME: " + jdks.myJdk + jdkPath = jdks.myJdk.getPath() + } + + class PrintJdk extends DefaultTask { + @Input + String jdkPath + + @TaskAction void print() { + println "JDK HOME: " + jdkPath } } """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy index 7f7922c0623d3..12d0ce41e105e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -250,6 +250,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { def "generates artifacts for shadowed elasticsearch plugin"() { given: + // we use the esplugin plugin in this test that is not configuration cache compatible yet + configurationCacheCompatible = false file('license.txt') << "License file" file('notice.txt') << "Notice file" buildFile << """ @@ -334,6 +336,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest { def "generates pom for elasticsearch plugin"() { given: + // we use the esplugin plugin in this test that is not configuration cache compatible yet + configurationCacheCompatible = false file('license.txt') << "License file" file('notice.txt') << "Notice file" buildFile << """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy index f3baf5910fca9..8686ad3df79ae 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy @@ -15,6 +15,8 @@ class ElasticsearchDistributionPluginFuncTest extends AbstractGradleFuncTest { def "copied modules are resolved from explodedBundleZip"() { given: + // we use the esplugin plugin in this test that is not configuration cache compatible yet + configurationCacheCompatible = false moduleSubProject() buildFile << """plugins { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy index 3ad8999f838e6..fed667b014c23 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy @@ -18,7 +18,6 @@ class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleInternalPlugin Class pluginClassUnderTest = LicenseHeadersPrecommitPlugin.class def setup() { - configurationCacheCompatible = true buildFile << """ apply plugin:'java' """ diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy index 19d5296b8defe..af5462630f55d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy @@ -24,13 +24,9 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl Class pluginClassUnderTest = TestingConventionsPrecommitPlugin.class - @ClassRule - @Shared - public TemporaryFolder repoFolder = new TemporaryFolder() - @Shared @ClassRule - public LocalRepositoryFixture repository = new LocalRepositoryFixture(repoFolder) + public LocalRepositoryFixture repository = new LocalRepositoryFixture() def setupSpec() { repository.generateJar('org.apache.lucene', 'tests.util', "1.0", @@ -45,7 +41,6 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl } def setup() { - configurationCacheCompatible = true repository.configureBuild(buildFile) } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy index 94100d499c236..11b32e108a54b 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy @@ -15,25 +15,24 @@ import net.bytebuddy.dynamic.DynamicType import net.bytebuddy.implementation.FixedValue import org.apache.logging.log4j.LogManager import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest +import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin +import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin import org.gradle.testkit.runner.TaskOutcome import static org.elasticsearch.gradle.fixtures.TestClasspathUtils.setupJarJdkClasspath -class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { +class ThirdPartyAuditTaskFuncTest extends AbstractGradleInternalPluginFuncTest { + + Class pluginClassUnderTest = ThirdPartyAuditPrecommitPlugin.class def setup() { buildFile << """ import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditPrecommitPlugin import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditTask - plugins { - id 'java' - // bring in build-tools onto the classpath - id 'elasticsearch.global-build-info' - } - - plugins.apply(ThirdPartyAuditPrecommitPlugin) + apply plugin:'java' group = 'org.elasticsearch' version = 'current' @@ -48,7 +47,7 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { mavenCentral() } - tasks.register("thirdPartyCheck", ThirdPartyAuditTask) { + tasks.named("thirdPartyAudit").configure { signatureFile = file('signature-file.txt') } """ @@ -58,6 +57,7 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { given: def group = "org.elasticsearch.gradle" generateDummyJars(group) + setupJarJdkClasspath(dir('local-repo/org/elasticsearch/elasticsearch-core/current/')) file('signature-file.txt') << "@defaultMessage non-public internal runtime class" buildFile << """ @@ -68,9 +68,9 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { } """ when: - def result = gradleRunner("thirdPartyCheck").build() + def result = gradleRunner("thirdPartyAudit").build() then: - result.task(":thirdPartyCheck").outcome == TaskOutcome.NO_SOURCE + result.task(":thirdPartyAudit").outcome == TaskOutcome.NO_SOURCE assertNoDeprecationWarning(result) } @@ -91,9 +91,9 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { } """ when: - def result = gradleRunner(":thirdPartyCheck").buildAndFail() + def result = gradleRunner(":thirdPartyAudit").buildAndFail() then: - result.task(":thirdPartyCheck").outcome == TaskOutcome.FAILED + result.task(":thirdPartyAudit").outcome == TaskOutcome.FAILED def output = normalized(result.getOutput()) assertOutputContains(output, """\ @@ -127,9 +127,9 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { } """ when: - def result = gradleRunner(":thirdPartyCheck").buildAndFail() + def result = gradleRunner(":thirdPartyAudit").buildAndFail() then: - result.task(":thirdPartyCheck").outcome == TaskOutcome.FAILED + result.task(":thirdPartyAudit").outcome == TaskOutcome.FAILED def output = normalized(result.getOutput()) assertOutputContains(output, """\ @@ -163,9 +163,9 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { } """ when: - def result = gradleRunner(":thirdPartyCheck").buildAndFail() + def result = gradleRunner(":thirdPartyAudit").buildAndFail() then: - result.task(":thirdPartyCheck").outcome == TaskOutcome.FAILED + result.task(":thirdPartyAudit").outcome == TaskOutcome.FAILED def output = normalized(result.getOutput()) assertOutputContains(output, """\ @@ -174,7 +174,7 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleFuncTest { """.stripIndent()) assertOutputContains(output, """\ * What went wrong: - Execution failed for task ':thirdPartyCheck'. + Execution failed for task ':thirdPartyAudit'. > Audit of third party dependencies failed: Jar Hell with the JDK: * diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy index ad8aad6f01baa..e70be2b4de4df 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy @@ -19,6 +19,8 @@ class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { def "yamlRestTest does nothing when there are no tests"() { given: + // RestIntegTestTask not cc compatible due to + configurationCacheCompatible = false buildFile << """ plugins { id 'elasticsearch.internal-yaml-rest-test' @@ -36,6 +38,8 @@ class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest { def "yamlRestTest executes and copies api and tests to correct source set"() { given: + // RestIntegTestTask not cc compatible due to + configurationCacheCompatible = false internalBuild() buildFile << """ apply plugin: 'elasticsearch.internal-yaml-rest-test' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy index 6db4a437a0296..e083dc6ff56d5 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy @@ -170,7 +170,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { file("/build/restResources/yamlTests/rest-api-spec/test/" + coreTest).getText("UTF-8") == "replacedWithValue" when: - result = gradleRunner("copyRestApiSpecsTask").build() + result = gradleRunner("copyRestApiSpecsTask", '--stacktrace').build() then: result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.UP_TO_DATE diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy index 35079b3d29848..acf86ddff73ff 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy @@ -28,6 +28,12 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { def READER = MAPPER.readerFor(ObjectNode.class) def WRITER = MAPPER.writerFor(ObjectNode.class) + def setup() { + // not cc compatible due to: + // 1. TestClustersPlugin not cc compatible due to listener registration + // 2. RestIntegTestTask not cc compatible due to + configurationCacheCompatible = false + } def "yamlRestTestVxCompatTest does nothing when there are no tests"() { given: subProject(":distribution:bwc:maintenance") << """ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java index 1c6831feab7b0..b7d88ce381437 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java @@ -32,11 +32,9 @@ // other packages (e.g org.elasticsearch.client) will point to server rather than // their own artifacts. public class ElasticsearchJavadocPlugin implements Plugin { - private Project project; @Override public void apply(Project project) { - this.project = project; // ignore missing javadocs project.getTasks().withType(Javadoc.class).configureEach(javadoc -> { // the -quiet here is because of a bug in gradle, in that adding a string option @@ -57,7 +55,7 @@ public void execute(Task task) { }); // Relying on configurations introduced by the java plugin - this.project.getPlugins().withType(JavaPlugin.class, javaPlugin -> project.afterEvaluate(project1 -> { + project.getPlugins().withType(JavaPlugin.class, javaPlugin -> project.afterEvaluate(project1 -> { var withShadowPlugin = project1.getPlugins().hasPlugin(ShadowPlugin.class); var compileClasspath = project.getConfigurations().getByName("compileClasspath"); @@ -67,25 +65,25 @@ public void execute(Task task) { var nonShadowedCompileClasspath = compileClasspath.copyRecursive( dependency -> shadowedDependencies.contains(dependency) == false ); - configureJavadocForConfiguration(false, nonShadowedCompileClasspath); - configureJavadocForConfiguration(true, shadowConfiguration); + configureJavadocForConfiguration(project, false, nonShadowedCompileClasspath); + configureJavadocForConfiguration(project, true, shadowConfiguration); } else { - configureJavadocForConfiguration(false, compileClasspath); + configureJavadocForConfiguration(project, false, compileClasspath); } })); } - private void configureJavadocForConfiguration(boolean shadow, Configuration configuration) { + private void configureJavadocForConfiguration(Project project, boolean shadow, Configuration configuration) { configuration.getAllDependencies() .stream() .sorted(Comparator.comparing(Dependency::getGroup)) .filter(d -> d instanceof ProjectDependency) .map(d -> (ProjectDependency) d) .filter(p -> p.getDependencyProject() != null) - .forEach(projectDependency -> configureDependency(shadow, projectDependency)); + .forEach(projectDependency -> configureDependency(project, shadow, projectDependency)); } - private void configureDependency(boolean shadowed, ProjectDependency dep) { + private void configureDependency(Project project, boolean shadowed, ProjectDependency dep) { var upstreamProject = dep.getDependencyProject(); if (shadowed) { /* diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index 06e85abd4289f..2bff852fe37e6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -156,7 +156,9 @@ private void logGlobalBuildInfo() { JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(BuildParams.getRuntimeJavaHome()); final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName(); final String runtimeJvmImplementationVersion = runtimeJvm.getImplementationVersion(); - LOGGER.quiet(" Runtime JDK Version : " + runtimeJvmImplementationVersion + " (" + runtimeJvmVendorDetails + ")"); + final String runtimeVersion = runtimeJvm.getRuntimeVersion(); + final String runtimeExtraDetails = runtimeJvmVendorDetails + ", " + runtimeVersion; + LOGGER.quiet(" Runtime JDK Version : " + runtimeJvmImplementationVersion + " (" + runtimeExtraDetails + ")"); LOGGER.quiet(" Runtime java.home : " + BuildParams.getRuntimeJavaHome()); LOGGER.quiet(" Gradle JDK Version : " + gradleJvmImplementationVersion + " (" + gradleJvmVendorDetails + ")"); LOGGER.quiet(" Gradle java.home : " + gradleJvm.getJavaHome()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java index 6f288e70a604b..6488f0d0dd30b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java @@ -86,9 +86,11 @@ public void execute(Task task) { DependencyHandler dependencies = project.getDependencies(); String checkstyleVersion = VersionProperties.getVersions().get("checkstyle"); - Provider dependencyProvider = project.provider(() -> "org.elasticsearch:build-conventions:" + project.getVersion()); + Provider conventionsDependencyProvider = project.provider( + () -> "org.elasticsearch:build-conventions:" + project.getVersion() + ); dependencies.add("checkstyle", "com.puppycrawl.tools:checkstyle:" + checkstyleVersion); - dependencies.addProvider("checkstyle", dependencyProvider, dep -> dep.setTransitive(false)); + dependencies.addProvider("checkstyle", conventionsDependencyProvider, dep -> dep.setTransitive(false)); project.getTasks().withType(Checkstyle.class).configureEach(t -> { t.dependsOn(copyCheckstyleConf); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index 99672c559da5e..0e2631cd7d8c5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -55,7 +55,7 @@ public TaskProvider createTask(Project project) { Configuration compileOnly = project.getConfigurations() .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); t.setClasspath(runtimeConfiguration.plus(compileOnly)); - t.setJarsToScan(runtimeConfiguration.fileCollection(dep -> { + t.getJarsToScan().from(runtimeConfiguration.fileCollection(dep -> { // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files, // or dependencies added as `files(...)`, we can't be sure if those are third party or not. // err on the side of scanning these to make sure we don't miss anything @@ -65,8 +65,8 @@ public TaskProvider createTask(Project project) { t.setJavaHome(Jvm.current().getJavaHome().getPath()); t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion)); t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile()); - t.setJdkJarHellClasspath(jdkJarHellConfig); - t.setForbiddenAPIsClasspath(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); + t.getJdkJarHellClasspath().from(jdkJarHellConfig); + t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); }); return audit; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 31cd17f6449a0..229184b05af6e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -14,6 +14,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.JavaVersion; import org.gradle.api.file.ArchiveOperations; +import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; @@ -55,7 +56,7 @@ import javax.inject.Inject; @CacheableTask -public class ThirdPartyAuditTask extends DefaultTask { +public abstract class ThirdPartyAuditTask extends DefaultTask { private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile( "WARNING: Class '(.*)' cannot be loaded \\(.*\\)\\. Please fix the classpath!" @@ -80,8 +81,6 @@ public class ThirdPartyAuditTask extends DefaultTask { private String javaHome; - private FileCollection jdkJarHellClasspath; - private final Property targetCompatibility; private final ArchiveOperations archiveOperations; @@ -94,10 +93,6 @@ public class ThirdPartyAuditTask extends DefaultTask { private FileCollection classpath; - private FileCollection jarsToScan; - - private FileCollection forbiddenApisClasspath; - @Inject public ThirdPartyAuditTask( ArchiveOperations archiveOperations, @@ -120,13 +115,7 @@ public Property getTargetCompatibility() { @InputFiles @PathSensitive(PathSensitivity.NAME_ONLY) - public FileCollection getForbiddenAPIsClasspath() { - return forbiddenApisClasspath; - } - - public void setForbiddenAPIsClasspath(FileCollection forbiddenApisClasspath) { - this.forbiddenApisClasspath = forbiddenApisClasspath; - } + public abstract ConfigurableFileCollection getForbiddenAPIsClasspath(); @InputFile @PathSensitive(PathSensitivity.NONE) @@ -161,13 +150,7 @@ public File getSuccessMarker() { // We use compile classpath normalization here because class implementation changes are irrelevant for the purposes of jdk jar hell. // We only care about the runtime classpath ABI here. @CompileClasspath - public FileCollection getJdkJarHellClasspath() { - return jdkJarHellClasspath.filter(File::exists); - } - - public void setJdkJarHellClasspath(FileCollection jdkJarHellClasspath) { - this.jdkJarHellClasspath = jdkJarHellClasspath; - } + abstract ConfigurableFileCollection getJdkJarHellClasspath(); public void ignoreMissingClasses(String... classesOrPackages) { if (classesOrPackages.length == 0) { @@ -207,13 +190,11 @@ public Set getMissingClassExcludes() { @Classpath @SkipWhenEmpty - public FileCollection getJarsToScan() { - return jarsToScan; - } + public abstract ConfigurableFileCollection getJarsToScan(); @TaskAction public void runThirdPartyAudit() throws IOException { - Set jars = jarsToScan.getFiles(); + Set jars = getJarsToScan().getFiles(); extractJars(jars, getJarExpandDir()); final String forbiddenApisOutput = runForbiddenAPIsCli(); final Set missingClasses = new TreeSet<>(); @@ -357,7 +338,7 @@ private String runForbiddenAPIsCli() throws IOException { if (javaHome != null) { spec.setExecutable(javaHome + "/bin/java"); } - spec.classpath(forbiddenApisClasspath, classpath); + spec.classpath(getForbiddenAPIsClasspath(), classpath); spec.jvmArgs("-Xmx1g"); spec.getMainClass().set("de.thetaphi.forbiddenapis.cli.CliMain"); spec.args("-f", getSignatureFile().getAbsolutePath(), "-d", getJarExpandDir(), "--allowmissingclasses"); @@ -383,8 +364,7 @@ private String runForbiddenAPIsCli() throws IOException { private Set runJdkJarHellCheck() throws IOException { ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); ExecResult execResult = execOperations.javaexec(spec -> { - spec.classpath(jdkJarHellClasspath, classpath); - + spec.classpath(getJdkJarHellClasspath(), classpath); spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS); spec.args(getJarExpandDir()); spec.setIgnoreExitValue(true); @@ -407,7 +387,4 @@ public void setClasspath(FileCollection classpath) { this.classpath = classpath; } - public void setJarsToScan(FileCollection jarsToScan) { - this.jarsToScan = jarsToScan; - } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java index 4f4b4ce33a8f8..2ca67134b8922 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java @@ -74,6 +74,7 @@ static String generateMigrationFile(QualifiedVersion version, String template, L bindings.put("deprecationsByNotabilityByArea", deprecationsByNotabilityByArea); bindings.put("isElasticsearchSnapshot", version.isSnapshot()); bindings.put("majorDotMinor", version.major() + "." + version.minor()); + bindings.put("majorDotMinorDotRevision", version.major() + "." + version.minor() + "." + version.revision()); bindings.put("majorMinor", String.valueOf(version.major()) + version.minor()); bindings.put("nextMajor", (version.major() + 1) + ".0"); bindings.put("version", version); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index 7827b34f57e43..093a7a19a42f9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -99,6 +99,7 @@ public Highlight getHighlight() { public void setHighlight(Highlight highlight) { this.highlight = highlight; + if (this.highlight != null) this.highlight.pr = this.pr; } public Breaking getBreaking() { @@ -160,6 +161,7 @@ public static class Highlight { private boolean notable; private String title; private String body; + private Integer pr; public boolean isNotable() { return notable; @@ -189,6 +191,10 @@ public String getAnchor() { return generatedAnchor(this.title); } + public Integer getPr() { + return pr; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java index 92de8c2ec1752..a562f0f3583f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ static String generateFile(QualifiedVersion version, String template, List> groupedHighlights = entries.stream() .map(ChangelogEntry::getHighlight) .filter(Objects::nonNull) + .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr)) .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java index 5c00e0428c9b7..1fbc367804ed4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.gradle.internal.test.rest; +import org.elasticsearch.gradle.internal.util.SerializableFunction; import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileCollection; @@ -29,7 +30,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.util.function.Function; import java.util.stream.Collectors; import javax.inject.Inject; @@ -54,8 +54,8 @@ public class CopyRestApiTask extends DefaultTask { private boolean skipHasRestTestCheck; private FileCollection config; private FileCollection additionalConfig; - private Function configToFileTree = FileCollection::getAsFileTree; - private Function additionalConfigToFileTree = FileCollection::getAsFileTree; + private SerializableFunction configToFileTree = FileCollection::getAsFileTree; + private SerializableFunction additionalConfigToFileTree = FileCollection::getAsFileTree; private final PatternFilterable patternSet; private final ProjectLayout projectLayout; @@ -176,11 +176,11 @@ public void setAdditionalConfig(FileCollection additionalConfig) { this.additionalConfig = additionalConfig; } - public void setConfigToFileTree(Function configToFileTree) { + public void setConfigToFileTree(SerializableFunction configToFileTree) { this.configToFileTree = configToFileTree; } - public void setAdditionalConfigToFileTree(Function additionalConfigToFileTree) { + public void setAdditionalConfigToFileTree(SerializableFunction additionalConfigToFileTree) { this.additionalConfigToFileTree = additionalConfigToFileTree; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java index 5cc68f8e73d45..9359272b29610 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java @@ -8,6 +8,7 @@ package org.elasticsearch.gradle.internal.test.rest; import org.apache.tools.ant.filters.ReplaceTokens; +import org.elasticsearch.gradle.internal.util.SerializableFunction; import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.FileCollection; @@ -29,7 +30,6 @@ import java.io.File; import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; import javax.inject.Inject; @@ -53,9 +53,9 @@ public class CopyRestTestsTask extends DefaultTask { private FileCollection coreConfig; private FileCollection xpackConfig; private FileCollection additionalConfig; - private Function coreConfigToFileTree = FileCollection::getAsFileTree; - private Function xpackConfigToFileTree = FileCollection::getAsFileTree; - private Function additionalConfigToFileTree = FileCollection::getAsFileTree; + private SerializableFunction coreConfigToFileTree = FileCollection::getAsFileTree; + private SerializableFunction xpackConfigToFileTree = FileCollection::getAsFileTree; + private SerializableFunction additionalConfigToFileTree = FileCollection::getAsFileTree; private final PatternFilterable corePatternSet; private final PatternFilterable xpackPatternSet; @@ -183,15 +183,15 @@ public void setAdditionalConfig(FileCollection additionalConfig) { this.additionalConfig = additionalConfig; } - public void setCoreConfigToFileTree(Function coreConfigToFileTree) { + public void setCoreConfigToFileTree(SerializableFunction coreConfigToFileTree) { this.coreConfigToFileTree = coreConfigToFileTree; } - public void setXpackConfigToFileTree(Function xpackConfigToFileTree) { + public void setXpackConfigToFileTree(SerializableFunction xpackConfigToFileTree) { this.xpackConfigToFileTree = xpackConfigToFileTree; } - public void setAdditionalConfigToFileTree(Function additionalConfigToFileTree) { + public void setAdditionalConfigToFileTree(SerializableFunction additionalConfigToFileTree) { this.additionalConfigToFileTree = additionalConfigToFileTree; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java new file mode 100644 index 0000000000000..1f2842ed396d2 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/SerializableFunction.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.util; + +import java.io.Serializable; +import java.util.function.Function; + +/** + * A functional interface that extends Function but also Serializable. + * + * Gradle configuration cache requires fields that represent a lambda to be serializable. + * */ +@FunctionalInterface +public interface SerializableFunction extends Function, Serializable {} diff --git a/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml b/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml index db6b025dea2dc..c4870cbf6e5c1 100644 --- a/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml +++ b/build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml @@ -16,7 +16,7 @@ - + diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc index 0eba0ecf88a74..75abce3b2b6ef 100644 --- a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc @@ -9,14 +9,16 @@ your application to {es} ${majorDotMinor}. See also <> and <>. <% if (isElasticsearchSnapshot) { %> -coming::[${version}] +coming::[${majorDotMinorDotRevision}] <% } %> [discrete] [[breaking-changes-${majorDotMinor}]] === Breaking changes <% if (breakingByNotabilityByArea.isEmpty()) { %> +// tag::notable-breaking-changes[] There are no breaking changes in {es} ${majorDotMinor}. +// end::notable-breaking-changes[] <% } else { %> The following changes in {es} ${majorDotMinor} might affect your applications and prevent them from operating normally. diff --git a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc index 8b8d99babbc7f..f07ba9c5d4db3 100644 --- a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc +++ b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc @@ -32,7 +32,7 @@ if (notableHighlights.isEmpty()) { %> <% for (highlight in notableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> // end::notable-highlights[] @@ -40,6 +40,6 @@ ${highlight.body.trim()} <% for (highlight in nonNotableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java index 0a478fa1a2500..7f510bef22661 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java @@ -60,31 +60,24 @@ public void generateFile_rendersCorrectMarkup() throws Exception { } private List getEntries() { - ChangelogEntry entry1 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight1 = new ChangelogEntry.Highlight(); - entry1.setHighlight(highlight1); - - highlight1.setNotable(true); - highlight1.setTitle("Notable release highlight number 1"); - highlight1.setBody("Notable release body number 1"); - - ChangelogEntry entry2 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight2 = new ChangelogEntry.Highlight(); - entry2.setHighlight(highlight2); - - highlight2.setNotable(true); - highlight2.setTitle("Notable release highlight number 2"); - highlight2.setBody("Notable release body number 2"); + ChangelogEntry entry1 = makeChangelogEntry(1, true); + ChangelogEntry entry2 = makeChangelogEntry(2, true); + ChangelogEntry entry3 = makeChangelogEntry(3, false); + // Return unordered list, to test correct re-ordering + return List.of(entry2, entry1, entry3); + } - ChangelogEntry entry3 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight3 = new ChangelogEntry.Highlight(); - entry3.setHighlight(highlight3); + private ChangelogEntry makeChangelogEntry(int pr, boolean notable) { + ChangelogEntry entry = new ChangelogEntry(); + entry.setPr(pr); + ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); + entry.setHighlight(highlight); - highlight3.setNotable(false); - highlight3.setTitle("Notable release highlight number 3"); - highlight3.setBody("Notable release body number 3"); + highlight.setNotable(notable); + highlight.setTitle("Notable release highlight number " + pr); + highlight.setBody("Notable release body number " + pr); - return List.of(entry1, entry2, entry3); + return entry; } private String getResource(String name) throws Exception { diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc index 9f0ff4d0dbfe8..0de9941327a66 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc @@ -9,7 +9,7 @@ your application to {es} 8.4. See also <> and <>. -coming::[8.4.0-SNAPSHOT] +coming::[8.4.0] [discrete] diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc index 7fee857cea210..a55a590a8bca5 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc @@ -21,12 +21,12 @@ Other versions: [discrete] [[notable_release_highlight_number_1]] -=== Notable release highlight number 1 +=== {es-pull}1[Notable release highlight number 1] Notable release body number 1 [discrete] [[notable_release_highlight_number_2]] -=== Notable release highlight number 2 +=== {es-pull}2[Notable release highlight number 2] Notable release body number 2 // end::notable-highlights[] @@ -34,6 +34,6 @@ Notable release body number 2 [discrete] [[notable_release_highlight_number_3]] -=== Notable release highlight number 3 +=== {es-pull}3[Notable release highlight number 3] Notable release body number 3 diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index c9c578fa5aa58..4e81c274f2dd4 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -33,7 +33,7 @@ bouncycastle=1.64 opensaml = 4.0.1 # test dependencies -randomizedrunner = 2.7.7 +randomizedrunner = 2.8.0 junit = 4.12 junit5 = 5.7.1 httpclient = 4.5.13 diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy index 169ea6334c9fd..5287d4a932587 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy @@ -25,6 +25,8 @@ import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.with class TestClustersPluginFuncTest extends AbstractGradleFuncTest { def setup() { + // TestClusterPlugin with adding task listeners is not cc compatible + configurationCacheCompatible = false buildFile << """ import org.elasticsearch.gradle.testclusters.TestClustersAware import org.elasticsearch.gradle.testclusters.ElasticsearchCluster diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy index dc0d266688f4b..d486e109a9307 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy @@ -18,6 +18,11 @@ import java.util.stream.Collectors class PluginBuildPluginFuncTest extends AbstractGradleFuncTest { + def setup() { + // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible + configurationCacheCompatible = false + } + def "can assemble plugin via #taskName"() { given: buildFile << """plugins { diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy index 0355eb43eb46d..8cae82cccdf43 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/reaper/ReaperPluginFuncTest.groovy @@ -23,9 +23,10 @@ class ReaperPluginFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.ReaperPlugin; import org.elasticsearch.gradle.util.GradleUtils; + def serviceProvider = GradleUtils.getBuildService(project.getGradle().getSharedServices(), ReaperPlugin.REAPER_SERVICE_NAME); + tasks.register("launchReaper") { doLast { - def serviceProvider = GradleUtils.getBuildService(project.getGradle().getSharedServices(), ReaperPlugin.REAPER_SERVICE_NAME); def reaper = serviceProvider.get() reaper.registerCommand('test', 'true') reaper.unregister('test') diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy index 09c078b490f48..2d6d122663da3 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/JavaRestTestPluginFuncTest.groovy @@ -14,6 +14,11 @@ import org.gradle.testkit.runner.TaskOutcome class JavaRestTestPluginFuncTest extends AbstractGradleFuncTest { + def setup() { + // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible + configurationCacheCompatible = false + } + def "declares default dependencies"() { given: buildFile << """ diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy index 7ff2b4f61b913..83c215ceea42d 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/YamlRestTestPluginFuncTest.groovy @@ -14,6 +14,11 @@ import org.gradle.testkit.runner.TaskOutcome class YamlRestTestPluginFuncTest extends AbstractGradleFuncTest { + def setup() { + // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible + configurationCacheCompatible = false + } + def "declares default dependencies"() { given: buildFile << """ diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 755c866dafe8c..9fbfe498c61bf 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -34,7 +34,7 @@ abstract class AbstractGradleFuncTest extends Specification { File propertiesFile File projectDir - boolean configurationCacheCompatible = false + boolean configurationCacheCompatible = true def setup() { projectDir = testProjectDir.root diff --git a/docs/changelog/86227.yaml b/docs/changelog/86227.yaml deleted file mode 100644 index d36a4b3ae23f0..0000000000000 --- a/docs/changelog/86227.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 86227 -summary: Upgrade to Lucene 9.2 snapshot -area: Search -type: upgrade -issues: [] diff --git a/docs/changelog/86852.yaml b/docs/changelog/86852.yaml deleted file mode 100644 index 23c51684996c9..0000000000000 --- a/docs/changelog/86852.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 86852 -summary: Upgrade to lucene snapshot 978eef5459c -area: Search -type: upgrade -issues: [] diff --git a/docs/changelog/87472.yaml b/docs/changelog/87472.yaml new file mode 100644 index 0000000000000..a943b44829bb8 --- /dev/null +++ b/docs/changelog/87472.yaml @@ -0,0 +1,5 @@ +pr: 87472 +summary: "Script: Metadata for Update context" +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/docs/changelog/88013.yaml b/docs/changelog/88013.yaml new file mode 100644 index 0000000000000..3a4533728db70 --- /dev/null +++ b/docs/changelog/88013.yaml @@ -0,0 +1,6 @@ +pr: 88013 +summary: Periodic warning for 1-node cluster w/ seed hosts +area: Cluster Coordination +type: enhancement +issues: + - 85222 diff --git a/docs/changelog/88015.yaml b/docs/changelog/88015.yaml new file mode 100644 index 0000000000000..d5b534c64690a --- /dev/null +++ b/docs/changelog/88015.yaml @@ -0,0 +1,6 @@ +pr: 88015 +summary: Retry after all S3 get failures that made progress +area: Snapshot/Restore +type: enhancement +issues: + - 87243 diff --git a/docs/changelog/88035.yaml b/docs/changelog/88035.yaml new file mode 100644 index 0000000000000..01152c930520a --- /dev/null +++ b/docs/changelog/88035.yaml @@ -0,0 +1,5 @@ +pr: 88035 +summary: Sort ingest pipeline stats by use +area: Stats +type: enhancement +issues: [] diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 0d72d22bc1f62..2a7a1b32131bc 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,9 +1,13 @@ include::migration_intro.asciidoc[] -* <> +* <> +* <> +* <> * <> * <> +include::migrate_8_4.asciidoc[] +include::migrate_8_3.asciidoc[] include::migrate_8_2.asciidoc[] include::migrate_8_1.asciidoc[] include::migrate_8_0.asciidoc[] diff --git a/docs/reference/migration/migrate_8_3.asciidoc b/docs/reference/migration/migrate_8_3.asciidoc new file mode 100644 index 0000000000000..3c6214c0372c5 --- /dev/null +++ b/docs/reference/migration/migrate_8_3.asciidoc @@ -0,0 +1,64 @@ +[[migrating-8.3]] +== Migrating to 8.3 +++++ +8.3 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.3. + +See also <> and <>. + +coming::[8.3.0-SNAPSHOT] + + + +[discrete] +[[breaking-changes-8.3]] +=== Breaking changes + +There are no breaking changes in {es} 8.3. + + + +[discrete] +[[deprecated-8.3]] +=== Deprecations + +The following functionality has been deprecated in {es} 8.3 +and will be removed in a future version. +While this won't have an immediate impact on your applications, +we strongly encourage you take the described steps to update your code +after upgrading to 8.3. + +To find out if you are using any deprecated functionality, +enable <>. + + +[discrete] +[[deprecations_83_cluster_and_node_setting]] +==== Cluster and node setting deprecations + +[[configuring_bind_dn_in_an_ldap_or_active_directory_ad_realm_without_corresponding_bind_password_deprecated]] +.Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password is deprecated +[%collapsible] +==== +*Details* + +For LDAP or AD authentication realms, setting a bind DN (via the +`xpack.security.authc.realms.ldap.*.bind_dn` realm setting) without a +bind password is a misconfiguration that may prevent successful +authentication to the node. In the next major release, nodes will fail +to start if a bind DN is specified without a password. + +*Impact* + +If you have a bind DN configured for an LDAP or AD authentication +realm, set a bind password for {ref}/ldap-realm.html#ldap-realm-configuration[LDAP] +or {ref}/active-directory-realm.html#ad-realm-configuration[Active Directory]. +Configuring a bind DN without a password generates a warning in the +deprecation logs. + +*Note:* This deprecation only applies if your current LDAP or AD +configuration specifies a bind DN without a password. This scenario is +unlikely, but might impact a small subset of users. +==== + diff --git a/docs/reference/migration/migrate_8_4.asciidoc b/docs/reference/migration/migrate_8_4.asciidoc new file mode 100644 index 0000000000000..d0a676c86aa0e --- /dev/null +++ b/docs/reference/migration/migrate_8_4.asciidoc @@ -0,0 +1,22 @@ +[[migrating-8.4]] +== Migrating to 8.4 +++++ +8.4 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.4. + +See also <> and <>. + +coming::[8.4.0] + + +[discrete] +[[breaking-changes-8.4]] +=== Breaking changes + +// tag::notable-breaking-changes[] +There are no breaking changes in {es} 8.4. +// end::notable-breaking-changes[] + diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc index e875fe61ee42d..c3410900896e6 100644 --- a/docs/reference/modules/discovery/discovery-settings.asciidoc +++ b/docs/reference/modules/discovery/discovery-settings.asciidoc @@ -201,6 +201,11 @@ Sets how long the master node waits for each cluster state update to be completely published to all nodes, unless `discovery.type` is set to `single-node`. The default value is `30s`. See <>. +`cluster.discovery_configuration_check.interval `:: +(<>) +Sets the interval of some checks that will log warnings about an +incorrect discovery configuration. The default value is `30s`. + `cluster.join_validation.cache_timeout`:: (<>) When a node requests to join the cluster, the elected master node sends it a diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index fd8aa061f3bc6..8e869e1f516b9 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,6 +6,8 @@ This section summarizes the changes in each release. +* <> +* <> * <> * <> * <> @@ -24,6 +26,8 @@ This section summarizes the changes in each release. -- +include::release-notes/8.4.0.asciidoc[] +include::release-notes/8.3.0.asciidoc[] include::release-notes/8.2.3.asciidoc[] include::release-notes/8.2.2.asciidoc[] include::release-notes/8.2.1.asciidoc[] diff --git a/docs/reference/release-notes/8.3.0.asciidoc b/docs/reference/release-notes/8.3.0.asciidoc new file mode 100644 index 0000000000000..a60e66e63239c --- /dev/null +++ b/docs/reference/release-notes/8.3.0.asciidoc @@ -0,0 +1,365 @@ +[[release-notes-8.3.0]] +== {es} version 8.3.0 + +coming[8.3.0] + +Also see <>. + +[[bug-8.3.0]] +[float] +=== Bug fixes + +Aggregations:: +* Allow `serial_diff` under `min_doc_count` aggs {es-pull}86401[#86401] +* Allow bucket paths to specify `_count` within a bucket {es-pull}85720[#85720] +* Fix a bug with flattened fields in terms aggregations {es-pull}87392[#87392] +* Fix flaky `top_metrics` test {es-pull}86582[#86582] (issue: {es-issue}86377[#86377]) +* Fix: check field existence before trying to merge running stats {es-pull}86926[#86926] +* Fix: ordering terms aggregation on top metrics null values {es-pull}85774[#85774] +* Serialize interval in auto date histogram aggregation {es-pull}85473[#85473] + +Audit:: +* Fix audit logging to consistently include port number in `origin.address` {es-pull}86732[#86732] +* Support removing ignore filters for audit logging {es-pull}87675[#87675] (issue: {es-issue}68588[#68588]) + +Authentication:: +* An authorized user can disable a user with same name but different realm {es-pull}86473[#86473] +* Fix clearing of `lastSuccessfulAuthCache` when clear all realm cache API is called {es-pull}86909[#86909] (issue: {es-issue}86650[#86650]) + +Authorization:: +* Fix resolution of wildcard application privileges {es-pull}87293[#87293] + +CAT APIs:: +* Get hidden indices stats in `GET _cat/shards` {es-pull}86601[#86601] (issue: {es-issue}84656[#84656]) + +CCR:: +* Prevent invalid datastream metadata when CCR follows a datastream with closed indices on the follower {es-pull}87076[#87076] (issue: {es-issue}87048[#87048]) +* Remove some blocking in CcrRepository {es-pull}87235[#87235] + +Cluster Coordination:: +* Add `master_timeout` support to voting config exclusions APIs {es-pull}86670[#86670] +* Small fixes to clear voting config excls API {es-pull}87828[#87828] + +Discovery-Plugins:: +* [discovery-gce] Fix initialisation of transport in FIPS mode {es-pull}85817[#85817] (issue: {es-issue}85803[#85803]) + +Distributed:: +* Enforce external id uniqueness during `DesiredNode` construction {es-pull}84227[#84227] + +Engine:: +* Fork to WRITE thread before failing shard in `updateCheckPoints` {es-pull}87458[#87458] (issue: {es-issue}87094[#87094]) +* Removing Blocking Wait for Close in `RecoverySourceHandler` {es-pull}86127[#86127] (issue: {es-issue}85839[#85839]) + +Features:: +* Fix 'autoGeneratedTimestamp should not be set externally' error when retrying IndexRequest {es-pull}86184[#86184] (issue: {es-issue}83927[#83927]) + +Geo:: +* Fix Geotile aggregations on `geo_shapes` for precision 0 {es-pull}87202[#87202] (issue: {es-issue}87201[#87201]) +* Fix `null_value` for array-valued `geo_point` fields {es-pull}85959[#85959] +* Guard for adding null value tags to vector tiles {es-pull}87051[#87051] +* Quantize geo queries to remove true negatives from search results {es-pull}85441[#85441] (issue: {es-issue}40891[#40891]) + +Highlighting:: +* `FastVectorHighlighter` should use `ValueFetchers` to load source data {es-pull}85815[#85815] (issues: {es-issue}75011[#75011], {es-issue}84690[#84690], {es-issue}82458[#82458], {es-issue}80895[#80895]) + +ILM+SLM:: +* Make the ILM Move to Error Step Batched {es-pull}85565[#85565] (issue: {es-issue}81880[#81880]) + +Indices APIs:: +* Make `GetIndexAction` cancellable {es-pull}87681[#87681] + +Infra/Circuit Breakers:: +* Make CBE message creation more robust {es-pull}87881[#87881] + +Infra/Core:: +* Adjust osprobe assertion for burst cpu {es-pull}86990[#86990] +* Clean up `DeflateCompressor` after exception {es-pull}87163[#87163] (issue: {es-issue}87160[#87160]) +* Error on direct creation of non-primary system index {es-pull}86707[#86707] +* Fix null message in output {es-pull}86981[#86981] +* Fix using `FilterOutputStream` without overriding bulk write {es-pull}86304[#86304] +* Hide system indices and their aliases in upgraded clusters {es-pull}87125[#87125] +* Refactor code to avoid JDK bug: JDK-8285835 {es-pull}86614[#86614] + +Infra/Logging:: +* Temporarily provide `SystemPropertiesPropertySource` {es-pull}87149[#87149] + +Infra/Node Lifecycle:: +* Upgrade folders after settings validation {es-pull}87319[#87319] + +Infra/Plugins:: +* Use Windows newlines when listing plugin information on Windows {es-pull}86408[#86408] (issue: {es-issue}86352[#86352]) + +Infra/REST API:: +* Fix min node version before state recovery {es-pull}86482[#86482] + +Infra/Scripting:: +* Allow to sort by script value using `SemVer` semantics {es-pull}85990[#85990] (issues: {es-issue}85989[#85989], {es-issue}82287[#82287]) +* Script: Fix setter shortcut for unbridged setters {es-pull}86868[#86868] +* Script: Load Whitelists as Resource {es-pull}87539[#87539] + +Infra/Settings:: +* Permit removal of archived index settings {es-pull}86107[#86107] + +Ingest:: +* Execute self-reference checks once per pipeline {es-pull}85926[#85926] (issue: {es-issue}85790[#85790]) + +Java Low Level REST Client:: +* Do not retry client requests when failing with `ContentTooLargeException` {es-pull}87248[#87248] (issue: {es-issue}86041[#86041]) + +License:: +* Consistent response for starting basic license {es-pull}86272[#86272] (issue: {es-issue}86244[#86244]) + +Machine Learning:: +* Fix ML task auditor exception early in cluster lifecycle {es-pull}87023[#87023] (issue: {es-issue}87002[#87002]) +* Fix `WordPiece` tokenization of unknown words with known subwords {es-pull}87510[#87510] +* Fix distribution change check for `change_point` aggregation {es-pull}86423[#86423] +* Fixes inference timeout handling bug that throws unexpected `NullPointerException` {es-pull}87533[#87533] +* Correct logic for restart from failover fine tuning hyperparameters for training classification and regression models {ml-pull}2251[#2251] +* Fix possible source of "x = NaN, distribution = class boost::math::normal_distribution<..." log errors training classification and regression models {ml-pull}2249[#2249] +* Fix some bugs affecting decision to stop optimizing hyperparameters for training classification and regression models {ml-pull}2259[#2259] +* Fix cause of "Must provide points at which to evaluate function" log error training classification and regression models {ml-pull}2268[#2268] +* Fix a source of "Discarding sample = nan, weights = ..." log errors for time series anomaly detection {ml-pull}2286[#2286] + +Mapping:: +* Don't run `include_in_parent` when in `copy_to` context {es-pull}87123[#87123] (issue: {es-issue}87036[#87036]) + +Network:: +* Reject `openConnection` attempt while closing {es-pull}86315[#86315] (issue: {es-issue}86249[#86249]) + +Recovery:: +* Fail shard if STARTED after master failover {es-pull}87451[#87451] (issue: {es-issue}87367[#87367]) + +SQL:: +* Fix FORMAT function to comply with Microsoft SQL Server specification {es-pull}86225[#86225] (issue: {es-issue}66560[#66560]) +* Implement binary format support for SQL clear cursor {es-pull}84230[#84230] (issue: {es-issue}53359[#53359]) + +Search:: +* Add status field to Multi Search Template Responses {es-pull}85496[#85496] (issue: {es-issue}83029[#83029]) +* Fields API to allow fetching values when `_source` is disabled {es-pull}87267[#87267] (issue: {es-issue}87072[#87072]) +* Fix `_terms_enum` on unconfigured `constant_keyword` {es-pull}86191[#86191] (issues: {es-issue}86187[#86187], {es-issue}86267[#86267]) +* Fix status code when open point in time without `keep_alive` {es-pull}87011[#87011] (issue: {es-issue}87003[#87003]) +* Handle empty point values in `DiskUsage` API {es-pull}87826[#87826] (issue: {es-issue}87761[#87761]) +* Make sure to rewrite explain query on coordinator {es-pull}87013[#87013] (issue: {es-issue}64281[#64281]) + +Security:: +* Make user and role name constraint consistent with max document ID {es-pull}86728[#86728] (issue: {es-issue}66020[#66020]) +* Security plugin close releasable realms {es-pull}87429[#87429] (issue: {es-issue}86286[#86286]) + +Snapshot/Restore:: +* DONE should mean fully processed in snapshot status {es-pull}86414[#86414] +* Distinguish missing and invalid repositories {es-pull}85551[#85551] (issue: {es-issue}85550[#85550]) +* Fork after calling `getRepositoryData` from `StoreRecovery` {es-pull}87264[#87264] (issue: {es-issue}87237[#87237]) +* Fork after calling `getRepositoryData` from `StoreRecovery` {es-pull}87254[#87254] (issue: {es-issue}87237[#87237]) +* Throw exception on illegal `RepositoryData` updates {es-pull}87654[#87654] +* Upgrade Azure SDK to 12.16.0 {es-pull}86135[#86135] + +Stats:: +* Run `TransportClusterInfoActions` on MANAGEMENT pool {es-pull}87679[#87679] + +TSDB:: +* TSDB: fix the time_series in order collect priority {es-pull}85526[#85526] +* TSDB: fix wrong initial value of tsidOrd in TimeSeriesIndexSearcher {es-pull}85713[#85713] (issue: {es-issue}85711[#85711]) + +Transform:: +* Fix transform `_start` permissions to use stored headers in the config {es-pull}86802[#86802] +* [Transforms] fix bug when unsetting retention policy {es-pull}87711[#87711] + +[[deprecation-8.3.0]] +[float] +=== Deprecations + +Authentication:: +* Configuring a bind DN in an LDAP or Active Directory (AD) realm without a corresponding bind password is deprecated {es-pull}85326[#85326] (issue: {es-issue}47191[#47191]) + +[[enhancement-8.3.0]] +[float] +=== Enhancements + +Aggregations:: +* Improve min and max performance while in a `random_sampler` aggregation {es-pull}85118[#85118] + +Authentication:: +* Support configurable claims in JWT Realm Tokens {es-pull}86533[#86533] +* Warn on user roles disabled due to licensing requirements for document or field level security {es-pull}85393[#85393] (issue: {es-issue}79207[#79207]) +* `TokenService` decode JWTs, change warn to debug {es-pull}86498[#86498] + +Authorization:: +* Add delete privilege to `kibana_system` for Synthetics {es-pull}85844[#85844] +* Authorize painless execute as index action when an index is specified {es-pull}85512[#85512] (issue: {es-issue}86428[#86428]) +* Better error message for run-as denials {es-pull}85501[#85501] (issue: {es-issue}72904[#72904]) +* Improve "Has Privilege" performance for boolean-only response {es-pull}86685[#86685] +* Relax restrictions for role names in roles API {es-pull}86604[#86604] (issue: {es-issue}86480[#86480]) +* [Osquery] Extend `kibana_system` role with an access to osquery_manager… {es-pull}86609[#86609] + +Autoscaling:: +* Add support for CPU ranges in desired nodes {es-pull}86434[#86434] + +Cluster Coordination:: +* Block joins while applier is busy {es-pull}84919[#84919] +* Compute master task batch summary lazily {es-pull}86210[#86210] +* Log `cluster.initial_master_nodes` at startup {es-pull}86101[#86101] +* Reduce resource needs of join validation {es-pull}85380[#85380] (issue: {es-issue}83204[#83204]) +* Report pending joins in `ClusterFormationFailureHelper` {es-pull}85635[#85635] +* Speed up map diffing (2) {es-pull}86375[#86375] + +Discovery-Plugins:: +* Remove redundant jackson dependencies from discovery-azure {es-pull}87898[#87898] + +Distributed:: +* Keep track of desired nodes cluster membership {es-pull}84165[#84165] + +Engine:: +* Cache immutable translog lastModifiedTime {es-pull}82721[#82721] (issue: {es-issue}82720[#82720]) +* Increase `force_merge` threadpool size based on the allocated processors {es-pull}87082[#87082] (issue: {es-issue}84943[#84943]) +* More optimal forced merges when max_num_segments is greater than 1 {es-pull}85065[#85065] + +Geo:: +* Support 'GeoJSON' in CartesianPoint for 'point' {es-pull}85442[#85442] +* Support geo label position as runtime field {es-pull}86154[#86154] +* Support geo label position through REST vector tiles API {es-pull}86458[#86458] (issue: {es-issue}86044[#86044]) + +Health:: +* Add a basic check for tier preference and allocation filter clashing {es-pull}85071[#85071] +* Add preflight checks to Health API to ensure health is obtainable {es-pull}86404[#86404] +* Add tier information on health api migrate tiers user actions {es-pull}87486[#87486] +* Health api add indicator doc links {es-pull}86904[#86904] (issue: {es-issue}86892[#86892]) +* Health api copy editing {es-pull}87010[#87010] +* Return a default user action if no actions could be determined {es-pull}87079[#87079] + +ILM+SLM:: +* Make the ILM and SLM `history_index_enabled` settings dynamic {es-pull}86493[#86493] + +Indices APIs:: +* Batch execute template and pipeline cluster state operations {es-pull}86017[#86017] + +Infra/Core:: +* Add mapping for tags for the elastic agent {es-pull}86298[#86298] +* Expand jar hell to include modules {es-pull}86622[#86622] +* Faster GET _cluster/settings API {es-pull}86405[#86405] (issue: {es-issue}82342[#82342]) +* Faster string writes by saving stream flushes {es-pull}86114[#86114] +* Fleet: Add `start_time` and `minimum_execution_duration` attributes to actions {es-pull}86167[#86167] +* Force property expansion for security policy {es-pull}87396[#87396] +* Refactor array part into a `BytesRefArray` which can be serialized and … {es-pull}85826[#85826] +* Speed up ip v4 parser {es-pull}86253[#86253] +* Use varhandles for primitive type conversion in more places {es-pull}85577[#85577] (issue: {es-issue}78823[#78823]) + +Infra/Scripting:: +* Script: add ability to alias classes in whitelist {es-pull}86899[#86899] + +Ingest:: +* Iteratively execute synchronous ingest processors {es-pull}84250[#84250] (issue: {es-issue}84274[#84274]) +* Skip `ensureNoSelfReferences` check in `IngestService` {es-pull}87337[#87337] + +License:: +* Initialize active realms without logging a message {es-pull}86134[#86134] (issue: {es-issue}81380[#81380]) + +Machine Learning:: +* A text categorization aggregation that works like ML categorization {es-pull}80867[#80867] +* Add new _infer endpoint for all supervised models and deprecate deployment infer api {es-pull}86361[#86361] +* Adds new `question_answering` NLP task for extracting answers to questions from a document {es-pull}85958[#85958] +* Adds start and end params to `_preview` and excludes cold/frozen tiers from unbounded previews {es-pull}86989[#86989] +* Adjust automatic JVM heap sizing for dedicated ML nodes {es-pull}86399[#86399] +* Replace the implementation of the `categorize_text` aggregation {es-pull}85872[#85872] +* Upgrade PyTorch to version 1.11 {ml-pull}2233[#2233], {ml-pull}2235[#2235],{ml-pull}2238[#2238] +* Upgrade zlib to version 1.2.12 on Windows {ml-pull}2253[#2253] +* Upgrade libxml2 to version 2.9.14 on Linux and Windows {ml-pull}2287[#2287] +* Improve time series model stability and anomaly scoring consistency for data + for which many buckets are empty {ml-pull}2267[#2267] +* Address root cause for actual equals typical equals zero anomalies {ml-pull}2270[#2270] +* Better handling of outliers in update immediately after detecting changes in time series {ml-pull}2280[#2280] + +Mapping:: +* Intern field names in Mappers {es-pull}86301[#86301] +* Replace BYTE_BLOCK_SIZE - 2 with indexWriter#MAX_TERM_LENGTH {es-pull}85518[#85518] + +Network:: +* Log node identity at startup {es-pull}85773[#85773] + +Search:: +* GeoBoundingBox query should work on bounding box with equal latitude or longitude {es-pull}85788[#85788] (issue: {es-issue}77717[#77717]) +* Improve error message for search API url parameters {es-pull}86984[#86984] (issue: {es-issue}79719[#79719]) + +Security:: +* Add run-as support for OAuth2 tokens {es-pull}86680[#86680] +* Relax username restrictions for User APIs {es-pull}86398[#86398] (issue: {es-issue}86326[#86326]) +* User Profile - Add hint support to SuggestProfiles API {es-pull}85890[#85890] +* User Profile - Add new action origin and internal user {es-pull}86026[#86026] +* User Profile - Support request cancellation on HTTP disconnect {es-pull}86332[#86332] +* User Profile - add caching for `hasPrivileges` check {es-pull}86543[#86543] + +Snapshot/Restore:: +* Add parameter to exclude indices in a snapshot from response {es-pull}86269[#86269] (issue: {es-issue}82937[#82937]) + +Stats:: +* Add documentation for "io_time_in_millis" {es-pull}84911[#84911] + +TLS:: +* Set `serverAuth` extended key usage for generated certificates and CSRs {es-pull}86311[#86311] (issue: {es-issue}81067[#81067]) + +TSDB:: +* Aggregation Execution Context add timestamp provider {es-pull}85850[#85850] + +Transform:: +* Prefer secondary auth headers for transforms {es-pull}86757[#86757] +* Support `range` aggregation in transform {es-pull}86501[#86501] + +[[feature-8.3.0]] +[float] +=== New features + +Authorization:: +* Has privileges API for profiles {es-pull}85898[#85898] + +Geo:: +* New geo_grid query to be used with geogrid aggregations {es-pull}86596[#86596] (issue: {es-issue}85727[#85727]) + +Health:: +* Add support for `impact_areas` to health impacts {es-pull}85830[#85830] (issue: {es-issue}85829[#85829]) +* Add troubleshooting guides to shards allocation actions {es-pull}87078[#87078] +* Adding potential impacts to remaining health indicators {es-pull}86197[#86197] +* Health api drill down {es-pull}85234[#85234] (issue: {es-issue}84793[#84793]) +* New service to keep track of the master history as seen from each node {es-pull}85941[#85941] +* Sorting impact index names by index priority {es-pull}85347[#85347] + +Mapping:: +* Add support for dots in field names for metrics usecases {es-pull}86166[#86166] (issue: {es-issue}63530[#63530]) +* Synthetic source {es-pull}85649[#85649] + +SQL:: +* SQ: Allow partial results in SQL queries {es-pull}85897[#85897] (issue: {es-issue}33148[#33148]) + +Search:: +* Snapshots as simple archives {es-pull}86261[#86261] (issue: {es-issue}81210[#81210]) + +TSDB:: +* TSDB: Implement downsampling on time-series indices {es-pull}85708[#85708] (issues: {es-issue}69799[#69799], {es-issue}65769[#65769]) + +[[upgrade-8.3.0]] +[float] +=== Upgrades + +Infra/CLI:: +* Upgrade procrun executables to 1.3.1 {es-pull}86710[#86710] + +Infra/Core:: +* Upgrade jackson to 2.13.2 {es-pull}86051[#86051] + +Ingest:: +* Upgrading to tika 2.4 {es-pull}86015[#86015] + +Network:: +* Upgrade to Netty 4.1.76 {es-pull}86252[#86252] + +Packaging:: +* Update Iron Bank base image to 8.6 {es-pull}86796[#86796] + +SQL:: +* Update dependency - JLine - to v 3.21.0 {es-pull}83767[#83767] (issue: {es-issue}83575[#83575]) + +Search:: +* Update to public lucene 9.2.0 release {es-pull}87162[#87162] + +Snapshot/Restore:: +* Upgrade GCS Plugin to 1.118.1 {es-pull}87800[#87800] + + diff --git a/docs/reference/release-notes/8.4.0.asciidoc b/docs/reference/release-notes/8.4.0.asciidoc new file mode 100644 index 0000000000000..fa51fbc255656 --- /dev/null +++ b/docs/reference/release-notes/8.4.0.asciidoc @@ -0,0 +1,181 @@ +[[release-notes-8.4.0]] +== {es} version 8.4.0 + +coming[8.4.0] + +Also see <>. + +[[bug-8.4.0]] +[float] +=== Bug fixes + +Aggregations:: +* Make the metric in the `buckets_path` parameter optional {es-pull}87220[#87220] (issue: {es-issue}72983[#72983]) + +Allocation:: +* Clamp auto-expand replicas to the closest value {es-pull}87505[#87505] (issue: {es-issue}84788[#84788]) + +Authentication:: +* Fix unique realm name check to cover default realms {es-pull}87999[#87999] + +Authorization:: +* Add rollover permissions for `remote_monitoring_agent` {es-pull}87717[#87717] (issue: {es-issue}84161[#84161]) + +Autoscaling:: +* Do not include desired nodes in snapshots {es-pull}87695[#87695] + +EQL:: +* Avoid attempting PIT close on PIT open failure {es-pull}87498[#87498] + +Health:: +* Using the correct connection to fetch remote master history {es-pull}87299[#87299] + +Highlighting:: +* Handle ordering in plain highlighter for multiple inputs {es-pull}87414[#87414] (issue: {es-issue}87210[#87210]) + +ILM+SLM:: +* Batch ILM move to retry step task update {es-pull}86759[#86759] + +Infra/Core:: +* Disallow three-digit minor and revision versions {es-pull}87338[#87338] + +Ingest:: +* Don't ignore pipeline for upserts in bulk api {es-pull}87719[#87719] (issue: {es-issue}87131[#87131]) +* Geoip processor should respect the `ignore_missing` in case of missing database {es-pull}87793[#87793] (issue: {es-issue}87345[#87345]) + +Machine Learning:: +* Improve trained model stats API performance {es-pull}87978[#87978] + +SQL:: +* Fix date range checks {es-pull}87151[#87151] (issue: {es-issue}77179[#77179]) + +Snapshot/Restore:: +* Use the provided SAS token without SDK sanitation that can produce invalid signatures {es-pull}88155[#88155] (issue: {es-issue}88140[#88140]) + +Transform:: +* Execute `_refresh` separately from DBQ, with system permissions {es-pull}88005[#88005] (issue: {es-issue}88001[#88001]) + +[[enhancement-8.4.0]] +[float] +=== Enhancements + +Aggregations:: +* Minor `RangeAgg` optimization {es-pull}86935[#86935] (issue: {es-issue}84262[#84262]) +* Speed counting filters/range/date_histogram aggs {es-pull}81322[#81322] +* Update bucket metric pipeline agg paths to allow intermediate single bucket and bucket qualified multi-bucket aggs {es-pull}85729[#85729] + +Allocation:: +* Add debug information to `ReactiveReason` about assigned and unassigned shards {es-pull}86132[#86132] (issue: {es-issue}85243[#85243]) +* Use desired nodes during data tier allocation decisions {es-pull}87735[#87735] + +Audit:: +* User Profile - audit support for security domain {es-pull}87097[#87097] + +Authorization:: +* App permissions with action patterns do not retrieve privileges {es-pull}85455[#85455] +* Cancellable Profile Has Privilege check {es-pull}87224[#87224] +* Return action denied error when user with insufficient privileges (`manage_own_api_key`) attempts a grant API key request {es-pull}87461[#87461] (issue: {es-issue}87438[#87438]) + +Autoscaling:: +* Add processors to autoscaling capacity response {es-pull}87895[#87895] +* Keep track of desired nodes status in cluster state {es-pull}87474[#87474] + +Cluster Coordination:: +* Expose segment details in PCSS debug log {es-pull}87412[#87412] +* Report overall mapping size in cluster stats {es-pull}87556[#87556] + +Data streams:: +* Give doc-value-only mappings to numeric fields on metrics templates {es-pull}87100[#87100] + +Distributed:: +* Make Desired Nodes API operator-only {es-pull}87778[#87778] (issue: {es-issue}87777[#87777]) + +FIPS:: +* Log warning when hash function used by cache is not recommended in FIPS mode {es-pull}86740[#86740] +* Log warning when hashers for stored API keys or service tokens are not compliant with FIPS {es-pull}87363[#87363] + +Geo:: +* Optimize geogrid aggregations for singleton points {es-pull}87439[#87439] +* Use a faster but less accurate log algorithm for computing Geotile Y coordinate {es-pull}87515[#87515] + +Health:: +* Adding a transport action to get cluster formation info {es-pull}87306[#87306] +* Adding additional capability to the `master_is_stable` health indicator service {es-pull}87482[#87482] +* Creating a transport action for the `CoordinationDiagnosticsService` {es-pull}87984[#87984] +* Move the master stability logic into its own service separate from the `HealthIndicatorService` {es-pull}87672[#87672] +* Remove cluster block preflight check from health api {es-pull}87520[#87520] (issue: {es-issue}87464[#87464]) + +Infra/Core:: +* Improve console exception messages {es-pull}87942[#87942] +* Stop making index read-only when executing force merge index lifecycle management action {es-pull}81162[#81162] (issue: {es-issue}81162[#81162]) +* Stream input and output support for optional collections {es-pull}88127[#88127] +* Update version of internal http client {es-pull}87491[#87491] + +Infra/Logging:: +* Catch an exception when formatting a string fails {es-pull}87132[#87132] + +Ingest:: +* Allow pipeline processor to ignore missing pipelines {es-pull}87354[#87354] +* Move the ingest attachment processor to the default distribution {es-pull}87989[#87989] +* Only perform `ensureNoSelfReferences` check during ingest when needed {es-pull}87352[#87352] (issue: {es-issue}87335[#87335]) +* Removing `BouncyCastle` dependencies from ingest-attachment plugin {es-pull}88031[#88031] + +Machine Learning:: +* Add authorization info to ML config listings {es-pull}87884[#87884] +* Expand allowed NER labels to be any I-O-B tagged labels {es-pull}87091[#87091] +* Improve scalability of NLP models {es-pull}87366[#87366] + +Mapping:: +* Speed up `NumberFieldMapper` {es-pull}85688[#85688] + +Monitoring:: +* JvmService use SingleObjectCache {es-pull}87236[#87236] + +Network:: +* Allow start cluster with unreachable remote clusters {es-pull}87298[#87298] + +Performance:: +* Warn about impact of large readahead on search {es-pull}88007[#88007] + +Query Languages:: +* Add support for VERSION field type in SQL and EQL {es-pull}87590[#87590] (issue: {es-issue}83375[#83375]) + +Rollup:: +* [TSDB] Add Kahan support to downsampling summation {es-pull}87554[#87554] + +SQL:: +* Implement support for partial search results in SQL CLI {es-pull}86982[#86982] (issue: {es-issue}86082[#86082]) + +Search:: +* Add mapping stats for indexed `dense_vectors` {es-pull}86859[#86859] + +Security:: +* Automatically close idle connections in OIDC back-channel {es-pull}87773[#87773] +* Support exists query for API key query {es-pull}87229[#87229] + +Snapshot/Restore:: +* Make snapshot deletes not block the repository during data blob deletes {es-pull}86514[#86514] +* Update HDFS Repository to HDFS 3.3.3 {es-pull}88039[#88039] + +Transform:: +* Add authorization info to transform config listings {es-pull}87570[#87570] +* Implement per-transform num_failure_retries setting {es-pull}87361[#87361] + +[[feature-8.4.0]] +[float] +=== New features + +Health:: +* Master stability health indicator part 1 (when a master has been seen recently) {es-pull}86524[#86524] + +Infra/Logging:: +* Stable logging API - the basic use case {es-pull}86612[#86612] + +[[upgrade-8.4.0]] +[float] +=== Upgrades + +Network:: +* Upgrade to Netty 4.1.77 {es-pull}86630[#86630] + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 25b0d6b322e6e..87a5f6420252e 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -1,6 +1,8 @@ [[release-highlights]] == What's new in {minor-version} +coming::[{minor-version}] + Here are the highlights of what's new and improved in {es} {minor-version}! ifeval::[\{release-state}\"!=\"unreleased\"] For detailed information about this release, see the <> and @@ -10,7 +12,8 @@ endif::[] // Add previous release to the list Other versions: -{ref-bare}/8.2/release-highlights.html[8.2] +{ref-bare}/8.3/release-highlights.html[8.3] +| {ref-bare}/8.2/release-highlights.html[8.2] | {ref-bare}/8.1/release-highlights.html[8.1] | {ref-bare}/8.0/release-highlights.html[8.0] @@ -24,8 +27,3 @@ Other versions: // end::notable-highlights[] -[discrete] -[[integrate_filtering_support_for_approximate_nearest_neighbor_search]] -=== Integrate filtering support for approximate nearest neighbor search -The {ref}/knn-search-api.html[_knn_search endpoint] now has a "filter" option that allows to return only the nearest documents that satisfy the provided filter - diff --git a/docs/reference/tab-widgets/troubleshooting/data/add-tier.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/add-tier.asciidoc index 9e90a9f1a38ca..63e35a0b09169 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/add-tier.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/add-tier.asciidoc @@ -49,7 +49,7 @@ setting: GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference?flat_settings ---- + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- @@ -101,7 +101,7 @@ GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference // TEST[continued] -The reponse will look like this: +The response will look like this: [source,console-result] ---- diff --git a/docs/reference/tab-widgets/troubleshooting/data/increase-cluster-shard-limit.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/increase-cluster-shard-limit.asciidoc index 168b9b61f0175..a3c4362ad5571 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/increase-cluster-shard-limit.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/increase-cluster-shard-limit.asciidoc @@ -113,7 +113,7 @@ GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference // TEST[continued] -The reponse will look like this: +The response will look like this: [source,console-result] ---- diff --git a/docs/reference/tab-widgets/troubleshooting/data/increase-tier-capacity.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/increase-tier-capacity.asciidoc index dbb5ab5f4341e..a1db93ddb22e0 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/increase-tier-capacity.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/increase-tier-capacity.asciidoc @@ -211,7 +211,7 @@ GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference // TEST[continued] -The reponse will look like this: +The response will look like this: [source,console-result] ---- diff --git a/docs/reference/tab-widgets/troubleshooting/data/start-ilm.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/start-ilm.asciidoc index d76bb3625ca14..47b7f7c8dc780 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/start-ilm.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/start-ilm.asciidoc @@ -29,7 +29,7 @@ image::images/kibana-console.png[{kib} Console,align="center"] POST _ilm/start ---- + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- @@ -47,7 +47,7 @@ The reponse will look like this: GET _ilm/status ---- + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- @@ -61,14 +61,14 @@ The reponse will look like this: // end::cloud[] // tag::self-managed[] -<>: +<> {ilm}: [source,console] ---- POST _ilm/start ---- -The reponse will look like this: +The response will look like this: [source,console-result] ---- @@ -86,7 +86,7 @@ Verify {ilm} is now running: GET _ilm/status ---- -The reponse will look like this: +The response will look like this: [source,console-result] ---- diff --git a/docs/reference/tab-widgets/troubleshooting/data/start-slm.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/start-slm.asciidoc index 70179b42efce6..b2c6d6a1926f0 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/start-slm.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/start-slm.asciidoc @@ -29,7 +29,7 @@ image::images/kibana-console.png[{kib} Console,align="center"] POST _slm/start ---- + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- @@ -46,7 +46,7 @@ The reponse will look like this: GET _slm/status ---- + -The reponse will look like this: +The response will look like this: + [source,console-result] ---- @@ -64,7 +64,7 @@ The reponse will look like this: ---- POST _slm/start ---- -The reponse will look like this: +The response will look like this: [source,console-result] ---- { @@ -77,7 +77,7 @@ Verify the {slm} is now running: ---- GET _slm/status ---- -The reponse will look like this: +The response will look like this: [source,console-result] ---- { diff --git a/docs/reference/tab-widgets/troubleshooting/data/total-shards-per-node.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/total-shards-per-node.asciidoc index 7388d81831e64..b84c97e659717 100644 --- a/docs/reference/tab-widgets/troubleshooting/data/total-shards-per-node.asciidoc +++ b/docs/reference/tab-widgets/troubleshooting/data/total-shards-per-node.asciidoc @@ -113,7 +113,7 @@ GET /my-index-000001/_settings/index.routing.allocation.include._tier_preference // TEST[continued] -The reponse will look like this: +The response will look like this: [source,console-result] ---- diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index fc17506555edb..1714717d0e6d3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -48,7 +48,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadata().size(), equalTo(1)); // the default version + assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2")); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1")); @@ -60,7 +60,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadata().size(), equalTo(1)); // the default version + assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", Integer.class), equalTo(1)); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("2")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index ea73f3c318d66..976f6bb25705b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -49,8 +49,10 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; /** diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.meta.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.meta.txt new file mode 100644 index 0000000000000..586094ef5251e --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.meta.txt @@ -0,0 +1,31 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. + +# This file only whitelists the script class and the Metadata class for update scripts. +# Due to name conflicts, this whitelist cannot be loaded with other scripts with conflicting Metadata types. + +class org.elasticsearch.script.UpdateScript$Metadata @alias[class="Metadata"] { + String getIndex() + String getId() + String getRouting() + long getVersion() + Op getOp() + void setOp(Op) + ZonedDateTime getTimestamp() + String getType() +} + +class org.elasticsearch.script.UpdateScript { + Metadata meta() +} + +class org.elasticsearch.script.field.Op { + Op NOOP + Op INDEX + Op DELETE + Op CREATE +} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml index a23a27a2e6578..7a761d34ee593 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml @@ -123,3 +123,47 @@ - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "Iterable object is self-referencing itself" } + +--- +"Script Update Metadata": + - skip: + version: " - 8.3.99" + reason: "update metadata introduced in 8.4.0" + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.bar = meta().id + '-extra'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.bar: 2-extra } + - match: { found: true } + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "meta().op = Op.DELETE" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + catch: missing + get: + index: test_1 + id: "2" + + - match: { found: false } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml index 559a54d28a19e..4145f6d7a7964 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml @@ -93,3 +93,97 @@ id: "4" - match: { _source.within_one_minute: true } + +--- +"Script Upsert Metadata": + - skip: + version: " - 8.3.99" + reason: "update metadata introduced in 8.4.0" + + - do: + catch: /type unavailable for insert/ + update: + index: test_1 + id: "1" + body: + script: + source: "ctx._source.foo = meta().type" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.foo = meta().index + '_1'; ctx._source.bar = 'nothing'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.foo: test_1_1 } + - match: { _source.bar: nothing } + + - do: + update: + index: test_1 + id: "3" + body: + script: + source: "meta().op = Op.NOOP; ctx._source.bar = 'skipped?'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + catch: missing + get: + index: test_1 + id: "3" + + - match: { found: false } + + - do: + update: + index: test_1 + id: "3" + body: + script: + source: "meta().op = Op.CREATE; ctx._source.bar = 'skipped?'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "3" + + - match: { found: true } + - match: { _source.bar: "skipped?" } + + # update + - do: + update: + index: test_1 + id: "2" + body: + script: + source: "ctx._source.bar = meta().type + '-extra'" + lang: "painless" + upsert: {} + scripted_upsert: true + + - do: + get: + index: test_1 + id: "2" + + - match: { _source.bar: _doc-extra } diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java index 3a6a7fcb6c42c..7814354680e99 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java @@ -50,7 +50,7 @@ protected T applyScript(Consumer> IndexRequest index = new IndexRequest("index").id("1").source(singletonMap("foo", "bar")); ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "id", 0); when(scriptService.compile(any(), eq(UpdateScript.CONTEXT))).thenReturn( - (params, ctx) -> new UpdateScript(Collections.emptyMap(), ctx) { + (params, md) -> new UpdateScript(Collections.emptyMap(), md) { @Override public void execute() { scriptBody.accept(getCtx()); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java index 05b512617ab3f..93e218c70048f 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RetryingInputStream.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.Version; import org.elasticsearch.core.IOUtils; @@ -44,12 +45,13 @@ class S3RetryingInputStream extends InputStream { private final String blobKey; private final long start; private final long end; - private final int maxAttempts; private final List failures; private S3ObjectInputStream currentStream; + private long currentStreamFirstOffset; private long currentStreamLastOffset; private int attempt = 1; + private int failuresAfterMeaningfulProgress = 0; private long currentOffset; private boolean closed; private boolean eof; @@ -68,7 +70,6 @@ class S3RetryingInputStream extends InputStream { } this.blobStore = blobStore; this.blobKey = blobKey; - this.maxAttempts = blobStore.getMaxRetries() + 1; this.failures = new ArrayList<>(MAX_SUPPRESSED_EXCEPTIONS); this.start = start; this.end = end; @@ -85,7 +86,8 @@ private void openStream() throws IOException { getObjectRequest.setRange(Math.addExact(start, currentOffset), end); } final S3Object s3Object = SocketAccess.doPrivileged(() -> clientReference.client().getObject(getObjectRequest)); - this.currentStreamLastOffset = Math.addExact(Math.addExact(start, currentOffset), getStreamLength(s3Object)); + this.currentStreamFirstOffset = Math.addExact(start, currentOffset); + this.currentStreamLastOffset = Math.addExact(currentStreamFirstOffset, getStreamLength(s3Object)); this.currentStream = s3Object.getObjectContent(); } catch (final AmazonClientException e) { if (e instanceof AmazonS3Exception amazonS3Exception) { @@ -160,31 +162,32 @@ private void ensureOpen() { } private void reopenStreamOrFail(IOException e) throws IOException { - if (attempt >= maxAttempts) { - logger.debug( - () -> format( - "failed reading [%s/%s] at offset [%s], attempt [%s] of [%s], giving up", - blobStore.bucket(), - blobKey, - start + currentOffset, - attempt, - maxAttempts - ), - e - ); - throw addSuppressedExceptions(e); + final int maxAttempts = blobStore.getMaxRetries() + 1; + + final long meaningfulProgressSize = Math.max(1L, blobStore.bufferSizeInBytes() / 100L); + final long currentStreamProgress = Math.subtractExact(Math.addExact(start, currentOffset), currentStreamFirstOffset); + if (currentStreamProgress >= meaningfulProgressSize) { + failuresAfterMeaningfulProgress += 1; } - logger.debug( - () -> format( - "failed reading [%s/%s] at offset [%s], attempt [%s] of [%s], retrying", - blobStore.bucket(), - blobKey, - start + currentOffset, - attempt, - maxAttempts - ), - e + final Supplier messageSupplier = () -> format( + """ + failed reading [%s/%s] at offset [%s]; this was attempt [%s] to read this blob which yielded [%s] bytes; in total \ + [%s] of the attempts to read this blob have made meaningful progress and do not count towards the maximum number of \ + retries; the maximum number of read attempts which do not make meaningful progress is [%s]""", + blobStore.bucket(), + blobKey, + start + currentOffset, + attempt, + currentStreamProgress, + failuresAfterMeaningfulProgress, + maxAttempts ); + if (attempt >= maxAttempts + failuresAfterMeaningfulProgress) { + final var finalException = addSuppressedExceptions(e); + logger.warn(messageSupplier, finalException); + throw finalException; + } + logger.debug(messageSupplier, e); attempt += 1; if (failures.size() < MAX_SUPPRESSED_EXCEPTIONS) { failures.add(e); diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index d48483f2e1427..d16e7187e088b 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -10,6 +10,8 @@ import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream; import com.amazonaws.util.Base16; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; import org.apache.http.HttpStatus; import org.elasticsearch.cluster.metadata.RepositoryMetadata; @@ -43,6 +45,7 @@ import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Locale; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -57,6 +60,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; /** * This class tests how a {@link S3BlobContainer} and its underlying AWS S3 client are retrying requests when reading or writing blobs. @@ -439,6 +443,79 @@ public void testWriteLargeBlobStreaming() throws Exception { assertEquals(blobSize, bytesReceived.get()); } + public void testReadRetriesAfterMeaningfulProgress() throws Exception { + final int maxRetries = between(0, 5); + final int bufferSizeBytes = scaledRandomIntBetween( + 0, + randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes())) + ); + final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, new ByteSizeValue(bufferSizeBytes)); + final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100); + + final byte[] bytes = randomBlobContent(); + + @SuppressForbidden(reason = "use a http server") + class FlakyReadHandler implements HttpHandler { + private int failuresWithoutProgress; + + @Override + public void handle(HttpExchange exchange) throws IOException { + Streams.readFully(exchange.getRequestBody()); + if (failuresWithoutProgress >= maxRetries) { + final int rangeStart = getRangeStart(exchange); + assertThat(rangeStart, lessThan(bytes.length)); + exchange.getResponseHeaders().add("Content-Type", bytesContentType()); + final var remainderLength = bytes.length - rangeStart; + exchange.sendResponseHeaders(HttpStatus.SC_OK, remainderLength); + exchange.getResponseBody() + .write( + bytes, + rangeStart, + remainderLength < meaningfulProgressBytes ? remainderLength : between(meaningfulProgressBytes, remainderLength) + ); + } else if (randomBoolean()) { + failuresWithoutProgress += 1; + exchange.sendResponseHeaders( + randomFrom( + HttpStatus.SC_INTERNAL_SERVER_ERROR, + HttpStatus.SC_BAD_GATEWAY, + HttpStatus.SC_SERVICE_UNAVAILABLE, + HttpStatus.SC_GATEWAY_TIMEOUT + ), + -1 + ); + } else if (randomBoolean()) { + final var bytesSent = sendIncompleteContent(exchange, bytes); + if (bytesSent < meaningfulProgressBytes) { + failuresWithoutProgress += 1; + } else { + exchange.getResponseBody().flush(); + } + } else { + failuresWithoutProgress += 1; + } + exchange.close(); + } + } + + httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_max_retries"), new FlakyReadHandler()); + + try (InputStream inputStream = blobContainer.readBlob("read_blob_max_retries")) { + final int readLimit; + final InputStream wrappedStream; + if (randomBoolean()) { + // read stream only partly + readLimit = randomIntBetween(0, bytes.length); + wrappedStream = Streams.limitStream(inputStream, readLimit); + } else { + readLimit = bytes.length; + wrappedStream = inputStream; + } + final byte[] bytesRead = BytesReference.toBytes(Streams.readFully(wrappedStream)); + assertArrayEquals(Arrays.copyOfRange(bytes, 0, readLimit), bytesRead); + } + } + /** * Asserts that an InputStream is fully consumed, or aborted, when it is closed */ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java index 07c1ddfe3280e..9694a11eb4d6e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java @@ -215,13 +215,7 @@ public void testJoinWaitsForClusterApplier() throws Exception { final var victimName = randomValueOtherThan(masterName, () -> randomFrom(internalCluster().getNodeNames())); logger.info("--> master [{}], victim [{}]", masterName, victimName); - // drop the victim from the cluster with a network disruption - final var masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterName); - masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(TransportService.class, victimName)); - logger.info("--> waiting for victim's departure"); - ensureStableCluster(2, masterName); - - // block the cluster applier thread on the victim + // block the cluster applier thread on the victim (we expect no further cluster state applications at this point) logger.info("--> blocking victim's applier service"); final var barrier = new CyclicBarrier(2); internalCluster().getInstance(ClusterService.class, victimName).getClusterApplierService().onNewClusterState("block", () -> { @@ -235,6 +229,12 @@ public void testJoinWaitsForClusterApplier() throws Exception { }, ActionListener.wrap(() -> {})); barrier.await(10, TimeUnit.SECONDS); + // drop the victim from the cluster with a network disruption + final var masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterName); + masterTransportService.addFailToSendNoConnectRule(internalCluster().getInstance(TransportService.class, victimName)); + logger.info("--> waiting for victim's departure"); + ensureStableCluster(2, masterName); + // verify that the victim sends no joins while the applier is blocked final var victimTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, victimName); victimTransportService.addSendBehavior((connection, requestId, action, request, options) -> { diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index a7fb33167dc23..d7bd5f2dd5058 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -275,7 +275,7 @@ exports org.elasticsearch.monitor.os; exports org.elasticsearch.monitor.process; exports org.elasticsearch.node; - exports org.elasticsearch.operator; + exports org.elasticsearch.immutablestate; exports org.elasticsearch.persistent; exports org.elasticsearch.persistent.decider; exports org.elasticsearch.plugins; diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 06b850815681e..1660c8a9d7c0b 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; @@ -33,6 +32,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.UpdateScript; +import org.elasticsearch.script.field.Op; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -87,25 +87,10 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult * Execute a scripted upsert, where there is an existing upsert document and a script to be executed. The script is executed and a new * Tuple of operation and updated {@code _source} is returned. */ - Tuple> executeScriptedUpsert(Map upsertDoc, Script script, LongSupplier nowInMillis) { - Map ctx = Maps.newMapWithExpectedSize(3); - // Tell the script that this is a create and not an update - ctx.put(ContextFields.OP, UpdateOpType.CREATE.toString()); - ctx.put(ContextFields.SOURCE, upsertDoc); - ctx.put(ContextFields.NOW, nowInMillis.getAsLong()); - ctx = executeScript(script, ctx); - - UpdateOpType operation = UpdateOpType.lenientFromString((String) ctx.get(ContextFields.OP), logger, script.getIdOrCode()); - @SuppressWarnings("unchecked") - Map newSource = (Map) ctx.get(ContextFields.SOURCE); - - if (operation != UpdateOpType.CREATE && operation != UpdateOpType.NONE) { - // Only valid options for an upsert script are "create" (the default) or "none", meaning abort upsert - logger.warn("Invalid upsert operation [{}] for script [{}], doing nothing...", operation, script.getIdOrCode()); - operation = UpdateOpType.NONE; - } - - return new Tuple<>(operation, newSource); + Tuple> executeScriptedUpsert(Script script, UpdateScript.Metadata metadata) { + // Tell the script that this is a create and not an update (insert from upsert) + UpdateScript.Metadata md = executeScript(script, metadata); + return new Tuple<>(lenientGetOp(md, logger, script.getIdOrCode()), md.getSource()); } /** @@ -120,14 +105,13 @@ Result prepareUpsert(ShardId shardId, UpdateRequest request, final GetResult get if (request.scriptedUpsert() && request.script() != null) { // Run the script to perform the create logic IndexRequest upsert = request.upsertRequest(); - Tuple> upsertResult = executeScriptedUpsert( - upsert.sourceAsMap(), + Tuple> upsertResult = executeScriptedUpsert( request.script, - nowInMillis + UpdateScript.insert(getResult.getIndex(), getResult.getId(), Op.CREATE, nowInMillis.getAsLong(), upsert.sourceAsMap()) ); switch (upsertResult.v1()) { case CREATE -> indexRequest = Requests.indexRequest(request.index()).source(upsertResult.v2()); - case NONE -> { + case NOOP -> { UpdateResponse update = new UpdateResponse( shardId, getResult.getId(), @@ -237,26 +221,24 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes final String routing = calculateRouting(getResult, currentRequest); final Tuple> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final XContentType updateSourceContentType = sourceAndContent.v1(); - final Map sourceAsMap = sourceAndContent.v2(); - - Map ctx = Maps.newMapWithExpectedSize(16); - ctx.put(ContextFields.OP, UpdateOpType.INDEX.toString()); // The default operation is "index" - ctx.put(ContextFields.INDEX, getResult.getIndex()); - ctx.put(ContextFields.TYPE, MapperService.SINGLE_MAPPING_NAME); - ctx.put(ContextFields.ID, getResult.getId()); - ctx.put(ContextFields.VERSION, getResult.getVersion()); - ctx.put(ContextFields.ROUTING, routing); - ctx.put(ContextFields.SOURCE, sourceAsMap); - ctx.put(ContextFields.NOW, nowInMillis.getAsLong()); - - ctx = executeScript(request.script, ctx); - UpdateOpType operation = UpdateOpType.lenientFromString((String) ctx.get(ContextFields.OP), logger, request.script.getIdOrCode()); - - @SuppressWarnings("unchecked") - final Map updatedSourceAsMap = (Map) ctx.get(ContextFields.SOURCE); + UpdateScript.Metadata md = executeScript( + request.script, + UpdateScript.update( + getResult.getIndex(), + getResult.getId(), + getResult.getVersion(), + routing, + Op.INDEX, // The default operation is "index" + nowInMillis.getAsLong(), + MapperService.SINGLE_MAPPING_NAME, + sourceAndContent.v2() + ) + ); + Op op = lenientGetOp(md, logger, request.script.getIdOrCode()); + final Map updatedSourceAsMap = md.getSource(); - switch (operation) { + switch (op) { case INDEX -> { final IndexRequest indexRequest = Requests.indexRequest(request.index()) .id(request.id()) @@ -307,17 +289,17 @@ Result prepareUpdateScriptRequest(ShardId shardId, UpdateRequest request, GetRes } } - private Map executeScript(Script script, Map ctx) { + private UpdateScript.Metadata executeScript(Script script, UpdateScript.Metadata metadata) { try { if (scriptService != null) { UpdateScript.Factory factory = scriptService.compile(script, UpdateScript.CONTEXT); - UpdateScript executableScript = factory.newInstance(script.getParams(), ctx); + UpdateScript executableScript = factory.newInstance(script.getParams(), metadata); executableScript.execute(); } } catch (Exception e) { throw new IllegalArgumentException("failed to execute script", e); } - return ctx; + return metadata; } /** @@ -405,42 +387,13 @@ public XContentType updateSourceContentType() { } } - /** - * After executing the script, this is the type of operation that will be used for subsequent actions. This corresponds to the "ctx.op" - * variable inside of scripts. - */ - enum UpdateOpType { - CREATE("create"), - INDEX("index"), - DELETE("delete"), - NONE("none"); - - private final String name; - - UpdateOpType(String name) { - this.name = name; - } - - public static UpdateOpType lenientFromString(String operation, Logger logger, String scriptId) { - switch (operation) { - case "create": - return UpdateOpType.CREATE; - case "index": - return UpdateOpType.INDEX; - case "delete": - return UpdateOpType.DELETE; - case "none": - return UpdateOpType.NONE; - default: - // TODO: can we remove this leniency yet?? - logger.warn("Used upsert operation [{}] for script [{}], doing nothing...", operation, scriptId); - return UpdateOpType.NONE; - } - } - - @Override - public String toString() { - return name; + protected Op lenientGetOp(UpdateScript.Metadata md, Logger logger, String scriptId) { + try { + return md.getOp(); + } catch (IllegalArgumentException err) { + // TODO: can we remove this leniency yet?? (this comment from 1907c466, April 2017 -@stu) + logger.warn("[{}] for script [{}], doing nothing...", err.getMessage(), scriptId); + return Op.NOOP; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 1ab8227a14eb9..9cd1416280937 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -92,6 +92,7 @@ import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_ID; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.gateway.ClusterStateUpdaters.hideStateIfNotRecovered; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; import static org.elasticsearch.monitor.StatusInfo.Status.UNHEALTHY; @@ -116,6 +117,13 @@ public class Coordinator extends AbstractLifecycleComponent implements ClusterSt Setting.Property.NodeScope ); + public static final Setting SINGLE_NODE_CLUSTER_SEED_HOSTS_CHECK_INTERVAL_SETTING = Setting.timeSetting( + "cluster.discovery_configuration_check.interval", + TimeValue.timeValueMillis(30000), + TimeValue.timeValueMillis(1), + Setting.Property.NodeScope + ); + public static final String COMMIT_STATE_ACTION_NAME = "internal:cluster/coordination/commit_state"; private final Settings settings; @@ -140,6 +148,9 @@ public class Coordinator extends AbstractLifecycleComponent implements ClusterSt private final SeedHostsResolver configuredHostsResolver; private final TimeValue publishTimeout; private final TimeValue publishInfoTimeout; + private final TimeValue singleNodeClusterSeedHostsCheckInterval; + @Nullable + private Scheduler.Cancellable singleNodeClusterChecker = null; private final PublicationTransportHandler publicationHandler; private final LeaderChecker leaderChecker; private final FollowersChecker followersChecker; @@ -218,6 +229,7 @@ public Coordinator( this.joinAccumulator = new InitialJoinAccumulator(); this.publishTimeout = PUBLISH_TIMEOUT_SETTING.get(settings); this.publishInfoTimeout = PUBLISH_INFO_TIMEOUT_SETTING.get(settings); + this.singleNodeClusterSeedHostsCheckInterval = SINGLE_NODE_CLUSTER_SEED_HOSTS_CHECK_INTERVAL_SETTING.get(settings); this.random = random; this.electionSchedulerFactory = new ElectionSchedulerFactory(settings, random, transportService.getThreadPool()); this.preVoteCollector = new PreVoteCollector( @@ -739,6 +751,38 @@ private void processJoinRequest(JoinRequest joinRequest, ActionListener jo } } + private void cancelSingleNodeClusterChecker() { + assert Thread.holdsLock(mutex) : "Coordinator mutex not held"; + if (singleNodeClusterChecker != null) { + singleNodeClusterChecker.cancel(); + singleNodeClusterChecker = null; + } + } + + private void checkSingleNodeCluster() { + if (applierState.nodes().size() > 1) { + return; + } + + if (DISCOVERY_SEED_HOSTS_SETTING.exists(settings)) { + if (DISCOVERY_SEED_HOSTS_SETTING.get(settings).isEmpty()) { + // For a single-node cluster, the only acceptable setting is an empty list. + return; + } else { + logger.warn( + """ + This node is a fully-formed single-node cluster with cluster UUID [{}], but it is configured as if to \ + discover other nodes and form a multi-node cluster via the [{}] setting. Fully-formed clusters do not \ + attempt to discover other nodes, and nodes with different cluster UUIDs cannot belong to the same cluster. \ + The cluster UUID persists across restarts and can only be changed by deleting the contents of the node's \ + data path(s). Remove the discovery configuration to suppress this message.""", + applierState.metadata().clusterUUID(), + DISCOVERY_SEED_HOSTS_SETTING.getKey() + "=" + DISCOVERY_SEED_HOSTS_SETTING.get(settings) + ); + } + } + } + void becomeCandidate(String method) { assert Thread.holdsLock(mutex) : "Coordinator mutex not held"; logger.debug( @@ -748,6 +792,7 @@ void becomeCandidate(String method) { mode, lastKnownLeader ); + cancelSingleNodeClusterChecker(); if (mode != Mode.CANDIDATE) { final Mode prevMode = mode; @@ -803,6 +848,13 @@ private void becomeLeader() { assert leaderChecker.leader() == null : leaderChecker.leader(); followersChecker.updateFastResponseState(getCurrentTerm(), mode); + + if (applierState.nodes().size() > 1) { + cancelSingleNodeClusterChecker(); + } else if (singleNodeClusterChecker == null) { + singleNodeClusterChecker = transportService.getThreadPool() + .scheduleWithFixedDelay(() -> { checkSingleNodeCluster(); }, this.singleNodeClusterSeedHostsCheckInterval, Names.SAME); + } } void becomeFollower(String method, DiscoveryNode leaderNode) { @@ -822,6 +874,7 @@ void becomeFollower(String method, DiscoveryNode leaderNode) { lastKnownLeader ); } + cancelSingleNodeClusterChecker(); final boolean restartLeaderChecker = (mode == Mode.FOLLOWER && Optional.of(leaderNode).equals(lastKnownLeader)) == false; @@ -1028,6 +1081,10 @@ assert getLocalNode().equals(applierState.nodes().getMasterNode()) : coordinationState.get().getLastAcceptedConfiguration() + " != " + coordinationState.get().getLastCommittedConfiguration(); + + if (coordinationState.get().getLastAcceptedState().nodes().size() == 1) { + assert singleNodeClusterChecker != null; + } } else if (mode == Mode.FOLLOWER) { assert coordinationState.get().electionWon() == false : getLocalNode() + " is FOLLOWER so electionWon() should be false"; assert lastKnownLeader.isPresent() && (lastKnownLeader.get().equals(getLocalNode()) == false); @@ -1045,6 +1102,7 @@ assert getLocalNode().equals(applierState.nodes().getMasterNode()) assert currentPublication.map(Publication::isCommitted).orElse(true); assert preVoteCollector.getLeader().equals(lastKnownLeader.get()) : preVoteCollector; assert clusterFormationFailureHelper.isRunning() == false; + assert singleNodeClusterChecker == null; } else { assert mode == Mode.CANDIDATE; assert joinAccumulator instanceof JoinHelper.CandidateJoinAccumulator; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateErrorMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateErrorMetadata.java new file mode 100644 index 0000000000000..bd96750565c7c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateErrorMetadata.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +/** + * A metadata class to hold error information about errors encountered + * while applying a cluster state update for a given namespace. + *

+ * This information is held by the {@link ImmutableStateMetadata} class. + */ +public record ImmutableStateErrorMetadata(Long version, ErrorKind errorKind, List errors) + implements + SimpleDiffable, + ToXContentFragment { + + static final ParseField ERRORS = new ParseField("errors"); + static final ParseField VERSION = new ParseField("version"); + static final ParseField ERROR_KIND = new ParseField("error_kind"); + + /** + * Constructs an immutable state error metadata + * + * @param version the metadata version of the content which failed to apply + * @param errorKind the kind of error we encountered while processing + * @param errors the list of errors encountered during parsing and validation of the immutable state content + */ + public ImmutableStateErrorMetadata {} + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(version); + out.writeString(errorKind.getKindValue()); + out.writeCollection(errors, StreamOutput::writeString); + } + + /** + * Reads an {@link ImmutableStateErrorMetadata} from a {@link StreamInput} + * + * @param in the {@link StreamInput} to read from + * @return {@link ImmutableStateErrorMetadata} + * @throws IOException + */ + public static ImmutableStateErrorMetadata readFrom(StreamInput in) throws IOException { + return new ImmutableStateErrorMetadata(in.readLong(), ErrorKind.of(in.readString()), in.readList(StreamInput::readString)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(VERSION.getPreferredName(), version); + builder.field(ERROR_KIND.getPreferredName(), errorKind.getKindValue()); + builder.stringListField(ERRORS.getPreferredName(), errors); + builder.endObject(); + return builder; + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "immutable_state_error_metadata", + (a) -> new ImmutableStateErrorMetadata((Long) a[0], ErrorKind.of((String) a[1]), (List) a[2]) + ); + + static { + PARSER.declareLong(constructorArg(), VERSION); + PARSER.declareString(constructorArg(), ERROR_KIND); + PARSER.declareStringArray(constructorArg(), ERRORS); + } + + /** + * Reads an {@link ImmutableStateErrorMetadata} from xContent + * + * @param parser {@link XContentParser} + * @return {@link ImmutableStateErrorMetadata} + */ + public static ImmutableStateErrorMetadata fromXContent(final XContentParser parser) { + return PARSER.apply(parser, null); + } + + /** + * Reads an {@link ImmutableStateErrorMetadata} {@link Diff} from {@link StreamInput} + * + * @param in the {@link StreamInput} to read the diff from + * @return a {@link Diff} of {@link ImmutableStateErrorMetadata} + * @throws IOException + */ + public static Diff readDiffFrom(StreamInput in) throws IOException { + return SimpleDiffable.readDiffFrom(ImmutableStateErrorMetadata::readFrom, in); + } + + /** + * Enum for kinds of errors we might encounter while processing immutable cluster state updates. + */ + public enum ErrorKind { + PARSING("parsing"), + VALIDATION("validation"), + TRANSIENT("transient"); + + private final String kind; + + ErrorKind(String kind) { + this.kind = kind; + } + + /** + * Returns the String value for this enum value + * + * @return the String value for the enum + */ + public String getKindValue() { + return kind; + } + + /** + * Helper method to construct {@link ErrorKind} from a String. + * + * The JDK default implementation throws incomprehensible error. + * @param kind String value + * @return {@link ErrorKind} + */ + public static ErrorKind of(String kind) { + for (var report : values()) { + if (report.kind.equals(kind)) { + return report; + } + } + throw new IllegalArgumentException("kind not supported [" + kind + "]"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateHandlerMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateHandlerMetadata.java new file mode 100644 index 0000000000000..4232b11ca2d2e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateHandlerMetadata.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Metadata class to hold a set of immutable keys in the cluster state, set by each {@link ImmutableClusterStateHandler}. + * + *

+ * Since we hold immutable metadata state for multiple namespaces, the same handler can appear in + * multiple namespaces. See {@link ImmutableStateMetadata} and {@link Metadata}. + */ +public record ImmutableStateHandlerMetadata(String name, Set keys) + implements + SimpleDiffable, + ToXContentFragment { + + static final ParseField KEYS = new ParseField("keys"); + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeCollection(keys, StreamOutput::writeString); + } + + /** + * Reads an {@link ImmutableStateHandlerMetadata} from a {@link StreamInput} + * + * @param in the {@link StreamInput} to read from + * @return {@link ImmutableStateHandlerMetadata} + * @throws IOException + */ + public static ImmutableStateHandlerMetadata readFrom(StreamInput in) throws IOException { + return new ImmutableStateHandlerMetadata(in.readString(), in.readSet(StreamInput::readString)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name()); + builder.stringListField(KEYS.getPreferredName(), keys().stream().sorted().toList()); // ordered keys for output consistency + builder.endObject(); + return builder; + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "immutable_state_handler_metadata", + false, + (a, name) -> new ImmutableStateHandlerMetadata(name, Set.copyOf((List) a[0])) + ); + + static { + PARSER.declareStringArray(optionalConstructorArg(), KEYS); + } + + /** + * Reads an {@link ImmutableStateHandlerMetadata} from xContent + * + * @param parser {@link XContentParser} + * @return {@link ImmutableStateHandlerMetadata} + * @throws IOException + */ + public static ImmutableStateHandlerMetadata fromXContent(XContentParser parser, String name) throws IOException { + return PARSER.apply(parser, name); + } + + /** + * Reads an {@link ImmutableStateHandlerMetadata} {@link Diff} from {@link StreamInput} + * + * @param in the {@link StreamInput} to read the diff from + * @return a {@link Diff} of {@link ImmutableStateHandlerMetadata} + * @throws IOException + */ + public static Diff readDiffFrom(StreamInput in) throws IOException { + return SimpleDiffable.readDiffFrom(ImmutableStateHandlerMetadata::readFrom, in); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadata.java new file mode 100644 index 0000000000000..f876cf1328c47 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadata.java @@ -0,0 +1,295 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.DiffableUtils; +import org.elasticsearch.cluster.SimpleDiffable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Metadata class that contains information about immutable cluster state set + * through file based settings or by modules/plugins. + * + *

+ * These types of cluster settings/entities can be read through the REST API, + * but can only be modified through a versioned 'operator mode' update, e.g. + * file based settings or module/plugin upgrade. + */ +public record ImmutableStateMetadata( + String namespace, + Long version, + Map handlers, + ImmutableStateErrorMetadata errorMetadata +) implements SimpleDiffable, ToXContentFragment { + + private static final ParseField VERSION = new ParseField("version"); + private static final ParseField HANDLERS = new ParseField("handlers"); + private static final ParseField ERRORS_METADATA = new ParseField("errors"); + + /** + * ImmutableStateMetadata contains information about immutable cluster settings. + * + *

+ * These settings cannot be updated by the end user and are set outside of the + * REST layer, e.g. through file based settings or by plugin/modules. + * + * @param namespace The namespace of the setting creator, e.g. file_settings, security plugin, etc. + * @param version The update version, must increase with each update + * @param handlers Per state update handler information on key set in by this update. These keys are validated at REST time. + * @param errorMetadata If the update failed for some reason, this is where we store the error information metadata. + */ + public ImmutableStateMetadata {} + + /** + * Creates a set intersection between cluster state keys set by a given {@link ImmutableClusterStateHandler} + * and the input set. + * + *

+ * This method is to be used to check if a REST action handler is allowed to modify certain cluster state. + * + * @param handlerName the name of the immutable state handler we need to check for keys + * @param modified a set of keys we want to see if we can modify. + * @return + */ + public Set conflicts(String handlerName, Set modified) { + ImmutableStateHandlerMetadata handlerMetadata = handlers.get(handlerName); + if (handlerMetadata == null || handlerMetadata.keys().isEmpty()) { + return Collections.emptySet(); + } + + Set intersect = new HashSet<>(handlerMetadata.keys()); + intersect.retainAll(modified); + return Collections.unmodifiableSet(intersect); + } + + /** + * Reads an {@link ImmutableStateMetadata} from a {@link StreamInput} + * + * @param in the {@link StreamInput} to read from + * @return {@link ImmutableStateMetadata} + * @throws IOException + */ + public static ImmutableStateMetadata readFrom(StreamInput in) throws IOException { + Builder builder = new Builder(in.readString()).version(in.readLong()); + + int handlersSize = in.readVInt(); + for (int i = 0; i < handlersSize; i++) { + ImmutableStateHandlerMetadata handler = ImmutableStateHandlerMetadata.readFrom(in); + builder.putHandler(handler); + } + + builder.errorMetadata(in.readOptionalWriteable(ImmutableStateErrorMetadata::readFrom)); + return builder.build(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(namespace); + out.writeLong(version); + out.writeCollection(handlers.values()); + out.writeOptionalWriteable(errorMetadata); + } + + /** + * Reads an {@link ImmutableStateMetadata} {@link Diff} from {@link StreamInput} + * + * @param in the {@link StreamInput} to read the diff from + * @return a {@link Diff} of {@link ImmutableStateMetadata} + * @throws IOException + */ + public static Diff readDiffFrom(StreamInput in) throws IOException { + return SimpleDiffable.readDiffFrom(ImmutableStateMetadata::readFrom, in); + } + + /** + * Empty {@link org.elasticsearch.cluster.DiffableUtils.MapDiff} helper for metadata backwards compatibility. + */ + public static final DiffableUtils.MapDiff> EMPTY_DIFF = + new DiffableUtils.MapDiff<>(null, null, List.of(), List.of(), List.of()) { + @Override + public Map apply(Map part) { + return part; + } + }; + + /** + * Convenience method for creating a {@link Builder} for {@link ImmutableStateMetadata} + * + * @param namespace the namespace under which we'll store the {@link ImmutableStateMetadata} + * @return {@link Builder} + */ + public static Builder builder(String namespace) { + return new Builder(namespace); + } + + /** + * Convenience method for creating a {@link Builder} for {@link ImmutableStateMetadata} + * + * @param namespace the namespace under which we'll store the {@link ImmutableStateMetadata} + * @param metadata an existing {@link ImmutableStateMetadata} + * @return {@link Builder} + */ + public static Builder builder(String namespace, ImmutableStateMetadata metadata) { + return new Builder(namespace, metadata); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(namespace()); + builder.field(VERSION.getPreferredName(), version); + builder.startObject(HANDLERS.getPreferredName()); + for (var i = handlers.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(); i.hasNext();) { + i.next().getValue().toXContent(builder, params); + } + builder.endObject(); + builder.field(ERRORS_METADATA.getPreferredName(), errorMetadata); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "immutable_state_metadata", + false, + (a, namespace) -> { + Map handlers = new HashMap<>(); + @SuppressWarnings("unchecked") + List handlersList = (List) a[1]; + handlersList.forEach(h -> handlers.put(h.name(), h)); + + return new ImmutableStateMetadata(namespace, (Long) a[0], Map.copyOf(handlers), (ImmutableStateErrorMetadata) a[2]); + } + ); + + static { + PARSER.declareLong(constructorArg(), VERSION); + PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, name) -> ImmutableStateHandlerMetadata.fromXContent(p, name), HANDLERS); + PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> ImmutableStateErrorMetadata.fromXContent(p), null, ERRORS_METADATA); + } + + /** + * Reads {@link ImmutableStateMetadata} from {@link XContentParser} + * + * @param parser {@link XContentParser} + * @return {@link ImmutableStateMetadata} + * @throws IOException + */ + public static ImmutableStateMetadata fromXContent(final XContentParser parser) throws IOException { + parser.nextToken(); + return PARSER.apply(parser, parser.currentName()); + } + + /** + * Builder class for {@link ImmutableStateMetadata} + */ + public static class Builder { + private final String namespace; + private Long version; + private Map handlers; + ImmutableStateErrorMetadata errorMetadata; + + /** + * Empty builder for ImmutableStateMetadata. + *

+ * The immutable metadata namespace is a required parameter + * + * @param namespace The namespace for this immutable metadata + */ + public Builder(String namespace) { + this.namespace = namespace; + this.version = 0L; + this.handlers = new HashMap<>(); + this.errorMetadata = null; + } + + /** + * Creates an immutable state metadata builder + * + * @param namespace the namespace for which we are storing metadata, e.g. file_settings + * @param metadata the previous metadata + */ + public Builder(String namespace, ImmutableStateMetadata metadata) { + this(namespace); + if (metadata != null) { + this.version = metadata.version; + this.handlers = new HashMap<>(metadata.handlers); + this.errorMetadata = metadata.errorMetadata; + } + } + + /** + * Stores the version for the immutable state metadata. + * + *

+ * Each new immutable cluster state update mode requires a version bump. + * The version increase doesn't have to be monotonic. + * + * @param version the new immutable state metadata version + * @return {@link Builder} + */ + public Builder version(Long version) { + this.version = version; + return this; + } + + /** + * Adds {@link ImmutableStateErrorMetadata} if we need to store error information about certain + * immutable state processing. + * + * @param errorMetadata {@link ImmutableStateErrorMetadata} + * @return {@link Builder} + */ + public Builder errorMetadata(ImmutableStateErrorMetadata errorMetadata) { + this.errorMetadata = errorMetadata; + return this; + } + + /** + * Adds an {@link ImmutableStateHandlerMetadata} for this {@link ImmutableStateMetadata}. + * + *

+ * The handler metadata is stored in a map, keyed off the {@link ImmutableStateHandlerMetadata} name. Previously + * stored {@link ImmutableStateHandlerMetadata} for a given name is overwritten. + * + * @param handler {@link ImmutableStateHandlerMetadata} + * @return {@link Builder} + */ + public Builder putHandler(ImmutableStateHandlerMetadata handler) { + this.handlers.put(handler.name(), handler); + return this; + } + + /** + * Builds an {@link ImmutableStateMetadata} from this builder. + * + * @return {@link ImmutableStateMetadata} + */ + public ImmutableStateMetadata build() { + return new ImmutableStateMetadata(namespace, version, Collections.unmodifiableMap(handlers), errorMetadata); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 0f672cd40b76f..e3b562c0e2b47 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -209,6 +209,7 @@ default boolean isRestorable() { private final ImmutableOpenMap> aliasedIndices; private final ImmutableOpenMap templates; private final ImmutableOpenMap customs; + private final Map immutableStateMetadata; private final transient int totalNumberOfShards; // Transient ? not serializable anyway? private final int totalOpenIndexShards; @@ -248,7 +249,8 @@ private Metadata( String[] visibleClosedIndices, SortedMap indicesLookup, Map mappingsByHash, - Version oldestIndexVersion + Version oldestIndexVersion, + Map immutableStateMetadata ) { this.clusterUUID = clusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; @@ -273,6 +275,7 @@ private Metadata( this.indicesLookup = indicesLookup; this.mappingsByHash = mappingsByHash; this.oldestIndexVersion = oldestIndexVersion; + this.immutableStateMetadata = immutableStateMetadata; } public Metadata withIncrementedVersion() { @@ -299,7 +302,8 @@ public Metadata withIncrementedVersion() { visibleClosedIndices, indicesLookup, mappingsByHash, - oldestIndexVersion + oldestIndexVersion, + immutableStateMetadata ); } @@ -359,7 +363,8 @@ public Metadata withLifecycleState(final Index index, final LifecycleExecutionSt visibleClosedIndices, indicesLookup, mappingsByHash, - oldestIndexVersion + oldestIndexVersion, + immutableStateMetadata ); } @@ -387,7 +392,8 @@ public Metadata withCoordinationMetadata(CoordinationMetadata coordinationMetada visibleClosedIndices, indicesLookup, mappingsByHash, - oldestIndexVersion + oldestIndexVersion, + immutableStateMetadata ); } @@ -970,6 +976,15 @@ public Map customs() { return this.customs; } + /** + * Returns the full {@link ImmutableStateMetadata} Map for all + * immutable state namespaces. + * @return a map of namespace to {@link ImmutableStateMetadata} + */ + public Map immutableStateMetadata() { + return this.immutableStateMetadata; + } + /** * The collection of index deletions in the cluster. */ @@ -1111,6 +1126,7 @@ private static class MetadataDiff implements Diff { private final Diff> indices; private final Diff> templates; private final Diff> customs; + private final Diff> immutableStateMetadata; MetadataDiff(Metadata before, Metadata after) { clusterUUID = after.clusterUUID; @@ -1123,12 +1139,19 @@ private static class MetadataDiff implements Diff { indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); + immutableStateMetadata = DiffableUtils.diff( + before.immutableStateMetadata, + after.immutableStateMetadata, + DiffableUtils.getStringKeySerializer() + ); } private static final DiffableUtils.DiffableValueReader INDEX_METADATA_DIFF_VALUE_READER = new DiffableUtils.DiffableValueReader<>(IndexMetadata::readFrom, IndexMetadata::readDiffFrom); private static final DiffableUtils.DiffableValueReader TEMPLATES_DIFF_VALUE_READER = new DiffableUtils.DiffableValueReader<>(IndexTemplateMetadata::readFrom, IndexTemplateMetadata::readDiffFrom); + private static final DiffableUtils.DiffableValueReader IMMUTABLE_DIFF_VALUE_READER = + new DiffableUtils.DiffableValueReader<>(ImmutableStateMetadata::readFrom, ImmutableStateMetadata::readDiffFrom); MetadataDiff(StreamInput in) throws IOException { clusterUUID = in.readString(); @@ -1145,6 +1168,15 @@ private static class MetadataDiff implements Diff { indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), INDEX_METADATA_DIFF_VALUE_READER); templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), TEMPLATES_DIFF_VALUE_READER); customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + immutableStateMetadata = DiffableUtils.readJdkMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + IMMUTABLE_DIFF_VALUE_READER + ); + } else { + immutableStateMetadata = ImmutableStateMetadata.EMPTY_DIFF; + } } @Override @@ -1161,6 +1193,9 @@ public void writeTo(StreamOutput out) throws IOException { indices.writeTo(out); templates.writeTo(out); customs.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + immutableStateMetadata.writeTo(out); + } } @Override @@ -1178,6 +1213,7 @@ public Metadata apply(Metadata part) { builder.indices(indices.apply(part.indices)); builder.templates(templates.apply(part.templates)); builder.customs(customs.apply(part.customs)); + builder.put(Collections.unmodifiableMap(immutableStateMetadata.apply(part.immutableStateMetadata))); return builder.build(); } } @@ -1219,7 +1255,12 @@ public static Metadata readFrom(StreamInput in) throws IOException { Custom customIndexMetadata = in.readNamedWriteable(Custom.class); builder.putCustom(customIndexMetadata.getWriteableName(), customIndexMetadata); } - + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + int immutableStateSize = in.readVInt(); + for (int i = 0; i < immutableStateSize; i++) { + builder.put(ImmutableStateMetadata.readFrom(in)); + } + } return builder.build(); } @@ -1246,6 +1287,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeCollection(templates.values()); VersionedNamedWriteable.writeVersionedWritables(out, customs); + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeCollection(immutableStateMetadata.values()); + } } public static Builder builder() { @@ -1280,6 +1324,8 @@ public static class Builder { private SortedMap previousIndicesLookup; + private final Map immutableStateMetadata; + // If this is set to false we can skip checking #mappingsByHash for unused entries in #build(). Used as an optimization to save // the rather expensive call to #purgeUnusedEntries when building from another instance and we know that no mappings can have // become unused because no indices were updated or removed from this builder in a way that would cause unused entries in @@ -1307,6 +1353,7 @@ public Builder() { this.previousIndicesLookup = metadata.indicesLookup; this.mappingsByHash = new HashMap<>(metadata.mappingsByHash); this.checkForUnusedMappings = false; + this.immutableStateMetadata = new HashMap<>(metadata.immutableStateMetadata); } private Builder(Map mappingsByHash) { @@ -1315,6 +1362,7 @@ private Builder(Map mappingsByHash) { aliasedIndices = ImmutableOpenMap.builder(); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); + immutableStateMetadata = new HashMap<>(); indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize previousIndicesLookup = null; this.mappingsByHash = new HashMap<>(mappingsByHash); @@ -1667,6 +1715,26 @@ public Builder customs(Map customs) { return this; } + /** + * Adds a map of namespace to {@link ImmutableStateMetadata} into the metadata builder + * @param immutableStateMetadata a map of namespace to {@link ImmutableStateMetadata} + * @return {@link Builder} + */ + public Builder put(Map immutableStateMetadata) { + this.immutableStateMetadata.putAll(immutableStateMetadata); + return this; + } + + /** + * Adds a {@link ImmutableStateMetadata} for a given namespace to the metadata builder + * @param metadata an {@link ImmutableStateMetadata} + * @return {@link Builder} + */ + public Builder put(ImmutableStateMetadata metadata) { + immutableStateMetadata.put(metadata.namespace(), metadata); + return this; + } + public Builder indexGraveyard(final IndexGraveyard indexGraveyard) { putCustom(IndexGraveyard.TYPE, indexGraveyard); return this; @@ -1868,7 +1936,8 @@ public Metadata build() { visibleClosedIndicesArray, indicesLookup, Collections.unmodifiableMap(mappingsByHash), - Version.fromId(oldestIndexVersionId) + Version.fromId(oldestIndexVersionId), + Collections.unmodifiableMap(immutableStateMetadata) ); } @@ -2167,6 +2236,12 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon } } + builder.startObject("immutable_state"); + for (ImmutableStateMetadata immutableStateMetadata : metadata.immutableStateMetadata().values()) { + immutableStateMetadata.toXContent(builder, params); + } + builder.endObject(); + builder.endObject(); } @@ -2210,6 +2285,10 @@ public static Metadata fromXContent(XContentParser parser) throws IOException { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); } + } else if ("immutable_state".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + builder.put(ImmutableStateMetadata.fromXContent(parser)); + } } else { try { Custom custom = parser.namedObject(Custom.class, currentFieldName, null); diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index b225f6fb4ca87..89d16c01df42e 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -492,6 +492,7 @@ public void apply(Settings value, Settings current, Settings previous) { ElectionSchedulerFactory.ELECTION_DURATION_SETTING, Coordinator.PUBLISH_TIMEOUT_SETTING, Coordinator.PUBLISH_INFO_TIMEOUT_SETTING, + Coordinator.SINGLE_NODE_CLUSTER_SEED_HOSTS_CHECK_INTERVAL_SETTING, JoinValidationService.JOIN_VALIDATION_CACHE_TIMEOUT_SETTING, FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING, FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING, diff --git a/server/src/main/java/org/elasticsearch/operator/OperatorHandler.java b/server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandler.java similarity index 66% rename from server/src/main/java/org/elasticsearch/operator/OperatorHandler.java rename to server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandler.java index 216b99858e5c8..f8428a9423c51 100644 --- a/server/src/main/java/org/elasticsearch/operator/OperatorHandler.java +++ b/server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandler.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.operator; +package org.elasticsearch.immutablestate; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -15,29 +15,29 @@ import java.util.Collections; /** - * OperatorHandler base interface used for implementing operator state actions. + * Base interface used for implementing 'operator mode' cluster state updates. * *

- * Updating cluster state in operator mode, for file based settings and modules/plugins, requires - * that we have a separate update handler interface that is different than REST handlers. This interface declares - * the basic contract for implementing cluster state update handlers in operator mode. + * Updating cluster state in immutable mode, for file based settings and modules/plugins, requires + * that we have a separate update handler interface that is different than the REST handlers. This interface class declares + * the basic contract for implementing cluster state update handlers that result in an immutable cluster state. *

*/ -public interface OperatorHandler { +public interface ImmutableClusterStateHandler { String CONTENT = "content"; /** * Unique identifier for the handler. * *

- * The operator handler name is a unique identifier that is matched to a section in a - * cluster state update content. The operator cluster state updates are done as a single + * The handler name is a unique identifier that is matched to a section in a + * cluster state update content. The immutable cluster state updates are done as a single * cluster state update and the cluster state is typically supplied as a combined content, * unlike the REST handlers. This name must match a desired content key name in the combined * cluster state update, e.g. "ilm" or "cluster_settings" (for persistent cluster settings update). *

* - * @return a String with the operator key name, e.g "ilm". + * @return a String with the handler name, e.g "ilm". */ String name(); @@ -45,9 +45,9 @@ public interface OperatorHandler { * The transformation method implemented by the handler. * *

- * The transform method of the operator handler should apply the necessary changes to + * The transform method of the handler should apply the necessary changes to * the cluster state as it normally would in a REST handler. One difference is that the - * transform method in an operator handler must perform all CRUD operations of the cluster + * transform method in an immutable state handler must perform all CRUD operations of the cluster * state in one go. For that reason, we supply a wrapper class to the cluster state called * {@link TransformState}, which contains the current cluster state as well as any previous keys * set by this handler on prior invocation. @@ -65,14 +65,14 @@ public interface OperatorHandler { * *

* Sometimes certain parts of the cluster state cannot be created/updated without previously - * setting other cluster state components, e.g. composable templates. Since the cluster state handlers - * are processed in random order by the OperatorClusterStateController, this method gives an opportunity - * to any operator handler to declare other operator handlers it depends on. Given dependencies exist, - * the OperatorClusterStateController will order those handlers such that the handlers that are dependent + * setting other cluster state components, e.g. composable templates. Since the immutable cluster state handlers + * are processed in random order by the ImmutableClusterStateController, this method gives an opportunity + * to any immutable handler to declare other immutable state handlers it depends on. Given dependencies exist, + * the ImmutableClusterStateController will order those handlers such that the handlers that are dependent * on are processed first. *

* - * @return a collection of operator handler names + * @return a collection of immutable state handler names */ default Collection dependencies() { return Collections.emptyList(); @@ -82,12 +82,12 @@ default Collection dependencies() { * Generic validation helper method that throws consistent exception for all handlers. * *

- * All implementations of OperatorHandler should call the request validate method, by calling this default - * implementation. To aid in any special validation logic that may need to be implemented by the operator handler + * All implementations of {@link ImmutableClusterStateHandler} should call the request validate method, by calling this default + * implementation. To aid in any special validation logic that may need to be implemented by the immutable cluster state handler * we provide this convenience method. *

* - * @param request the master node request that we base this operator handler on + * @param request the master node request that we base this immutable state handler on */ default void validate(MasterNodeRequest request) { ActionRequestValidationException exception = request.validate(); diff --git a/server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerProvider.java b/server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerProvider.java new file mode 100644 index 0000000000000..555c278182949 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerProvider.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.immutablestate; + +import java.util.Collection; + +/** + * SPI service interface for supplying {@link ImmutableClusterStateHandler} implementations to Elasticsearch + * from plugins/modules. + */ +public interface ImmutableClusterStateHandlerProvider { + /** + * Returns a list of {@link ImmutableClusterStateHandler} implementations that a module/plugin supplies. + * @see ImmutableClusterStateHandler + * + * @return a list of ${@link ImmutableClusterStateHandler}s + */ + Collection> handlers(); +} diff --git a/server/src/main/java/org/elasticsearch/operator/TransformState.java b/server/src/main/java/org/elasticsearch/immutablestate/TransformState.java similarity index 69% rename from server/src/main/java/org/elasticsearch/operator/TransformState.java rename to server/src/main/java/org/elasticsearch/immutablestate/TransformState.java index a68ce7e1f2290..4472bb1d18dc7 100644 --- a/server/src/main/java/org/elasticsearch/operator/TransformState.java +++ b/server/src/main/java/org/elasticsearch/immutablestate/TransformState.java @@ -6,15 +6,15 @@ * Side Public License, v 1. */ -package org.elasticsearch.operator; +package org.elasticsearch.immutablestate; import org.elasticsearch.cluster.ClusterState; import java.util.Set; /** - * A {@link ClusterState} wrapper used by the OperatorClusterStateController to pass the - * current state as well as previous keys set by an {@link OperatorHandler} to each transform + * A {@link ClusterState} wrapper used by the ImmutableClusterStateController to pass the + * current state as well as previous keys set by an {@link ImmutableClusterStateHandler} to each transform * step of the cluster state update. * */ diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java index 68d779be37812..7adc51c343dc4 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java @@ -47,7 +47,7 @@ class IngestSourceAndMetadata extends AbstractMap implements Met /** * map of key to validating function. Should throw {@link IllegalArgumentException} on invalid value */ - static final Map> VALIDATORS = Map.of( + protected static final Map> VALIDATORS = Map.of( IngestDocument.Metadata.INDEX.getFieldName(), IngestSourceAndMetadata::stringValidator, IngestDocument.Metadata.ID.getFieldName(), @@ -97,7 +97,7 @@ class IngestSourceAndMetadata extends AbstractMap implements Met * @param validators validators to run on metadata map, if a key is in this map, the value is stored in metadata. * if null, use the default validators from {@link #VALIDATORS} */ - IngestSourceAndMetadata( + protected IngestSourceAndMetadata( Map source, Map metadata, ZonedDateTime timestamp, diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java index e8478af2d2585..3b320d8c71259 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestStats.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestStats.java @@ -38,8 +38,18 @@ public class IngestStats implements Writeable, ToXContentFragment { */ public IngestStats(Stats totalStats, List pipelineStats, Map> processorStats) { this.totalStats = totalStats; - this.pipelineStats = pipelineStats; + this.pipelineStats = pipelineStats.stream().sorted((p1, p2) -> { + final IngestStats.Stats p2Stats = p2.stats; + final IngestStats.Stats p1Stats = p1.stats; + final int ingestTimeCompare = Long.compare(p2Stats.ingestTimeInMillis, p1Stats.ingestTimeInMillis); + if (ingestTimeCompare == 0) { + return Long.compare(p2Stats.ingestCount, p1Stats.ingestCount); + } else { + return ingestTimeCompare; + } + }).toList(); this.processorStats = processorStats; + } /** diff --git a/server/src/main/java/org/elasticsearch/operator/OperatorHandlerProvider.java b/server/src/main/java/org/elasticsearch/operator/OperatorHandlerProvider.java deleted file mode 100644 index 0f168ea6e4d61..0000000000000 --- a/server/src/main/java/org/elasticsearch/operator/OperatorHandlerProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.operator; - -import java.util.Collection; - -/** - * SPI service interface for supplying OperatorHandler implementations to Elasticsearch - * from plugins/modules. - */ -public interface OperatorHandlerProvider { - /** - * Returns a list of OperatorHandler implementations that a module/plugin supplies. - * @see OperatorHandler - * - * @return a list of ${@link OperatorHandler}s - */ - Collection> handlers(); -} diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 59b39fb091ce9..058885de79f81 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -52,6 +52,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.ServiceLoader; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -279,7 +280,6 @@ private Map loadBundles(Set bundles) { // package-private for test visibility static void loadExtensions(Collection plugins) { - Map> extendingPluginsByName = plugins.stream() .flatMap(t -> t.descriptor().getExtendedPlugins().stream().map(extendedPlugin -> Tuple.tuple(extendedPlugin, t.instance()))) .collect(Collectors.groupingBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toList()))); @@ -293,6 +293,28 @@ static void loadExtensions(Collection plugins) { } } + /** + * SPI convenience method that uses the {@link ServiceLoader} JDK class to load various SPI providers + * from plugins/modules. + *

+ * For example: + * + * 

+     * var pluginHandlers = pluginsService.loadServiceProviders(OperatorHandlerProvider.class);
+     *
+ * @param service A templated service class to look for providers in plugins + * @return an immutable {@link List} of discovered providers in the plugins/modules + */ + public List loadServiceProviders(Class service) { + List result = new ArrayList<>(); + + for (LoadedPlugin pluginTuple : plugins()) { + ServiceLoader.load(service, pluginTuple.loader()).iterator().forEachRemaining(c -> result.add(c)); + } + + return Collections.unmodifiableList(result); + } + private static void loadExtensionsForPlugin(ExtensiblePlugin extensiblePlugin, List extendingPlugins) { ExtensiblePlugin.ExtensionLoader extensionLoader = new ExtensiblePlugin.ExtensionLoader() { @Override diff --git a/server/src/main/java/org/elasticsearch/script/UpdateScript.java b/server/src/main/java/org/elasticsearch/script/UpdateScript.java index 578e2fa7f29b1..21162f2e674b7 100644 --- a/server/src/main/java/org/elasticsearch/script/UpdateScript.java +++ b/server/src/main/java/org/elasticsearch/script/UpdateScript.java @@ -1,4 +1,3 @@ - /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -9,10 +8,20 @@ package org.elasticsearch.script; +import org.elasticsearch.script.field.MapBackedMetadata; +import org.elasticsearch.script.field.Op; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; import java.util.Map; /** - * An update script. + * A script used in the update API */ public abstract class UpdateScript { @@ -24,12 +33,11 @@ public abstract class UpdateScript { /** The generic runtime parameters for the script. */ private final Map params; - /** The update context for the script. */ - private final Map ctx; + private final Metadata metadata; - public UpdateScript(Map params, Map ctx) { + public UpdateScript(Map params, Metadata metadata) { this.params = params; - this.ctx = ctx; + this.metadata = metadata; } /** Return the parameters for this script. */ @@ -39,12 +47,177 @@ public Map getParams() { /** Return the update context for this script. */ public Map getCtx() { - return ctx; + return metadata != null ? metadata.store.getMap() : null; + } + + /** return the metadata for this script */ + public Metadata meta() { + return metadata; } public abstract void execute(); public interface Factory { - UpdateScript newInstance(Map params, Map ctx); + UpdateScript newInstance(Map params, Metadata metadata); + } + + public static Metadata insert(String index, String id, Op op, long epochMilli, Map source) { + return new Metadata(index, id, op, epochMilli, source); + } + + public static Metadata update( + String index, + String id, + Long version, + String routing, + Op op, + long epochMilli, + String type, + Map source + ) { + return new Metadata(index, id, version, routing, op, epochMilli, type, source); + } + + /** + * Metadata for update scripts. Update scripts have different metadata available for updates to existing + * documents and for inserts (via upsert). + * + * Metadata unique to updates: + * _routing, _version and _type + * + * Common metadata: + * _index, _id, _now (timestamp) + * _op, which can be NOOP ("none"), INDEX, DELETE, CREATE for update and CREATE or "none" (Op.NOOP) for insert + */ + public static class Metadata { + private static final String TIMESTAMP = "_now"; + private static final String TYPE = "_type"; + private static final String LEGACY_NOOP_STRING = "none"; + private final MapBackedMetadata store; + private final ZonedDateTime timestamp; + + // insertions via upsert have fewer fields available. However, to allow the same script to be used on insertions + // and updates, the same Metadata class is used with the isInsert flag differentiating the two behaviors. + private final boolean isInsert; + protected final EnumSet validOps; + protected static final EnumSet INSERT_VALID_OPS = EnumSet.of(Op.NOOP, Op.CREATE); + protected static final EnumSet UPDATE_VALID_OPS = EnumSet.of(Op.NOOP, Op.INDEX, Op.DELETE); + + /** + * Insert via Upsert + */ + private Metadata(String index, String id, Op op, long epochMilli, Map source) { + this.store = new MapBackedMetadata(5).setIndex(index).setId(id).setOp(op).set(TIMESTAMP, epochMilli).setSource(source); + this.timestamp = ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC); + this.isInsert = true; + this.validOps = INSERT_VALID_OPS; + } + + /** + * Scripted Update and Update via Upsert + */ + private Metadata( + String index, + String id, + Long version, + String routing, + Op op, + long epochMilli, + String type, + Map source + ) { + this.store = new MapBackedMetadata(16).setIndex(index) + .setId(id) + .setVersion(version) + .setRouting(routing) + .setOp(op) + .set(TIMESTAMP, epochMilli) + .set(TYPE, type) + .setSource(source); + this.timestamp = ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC); + this.isInsert = false; + this.validOps = UPDATE_VALID_OPS; + } + + public String getIndex() { + return store.getIndex(); + } + + public String getId() { + return store.getId(); + } + + public String getRouting() { + if (isInsert) { + throw new IllegalStateException("routing unavailable for insert"); + } + return store.getRouting(); + } + + public long getVersion() { + if (isInsert) { + throw new IllegalStateException("version unavailable for inserts"); + } + Long version = store.getVersion(); + if (version == null) { + return Long.MIN_VALUE; + } + return version; + } + + public boolean hasVersion() { + if (isInsert) { + throw new IllegalStateException("version unavailable for inserts"); + } + return store.getVersion() == null; + } + + public List validOps() { + List enumOps = validOps.stream().map(Op::getName).toList(); + List ops = new ArrayList<>(enumOps.size() + 1); + ops.add("none"); + ops.addAll(enumOps); + return ops; + } + + public Op getOp() { + MapBackedMetadata.RawOp raw = store.getOp(); + if (raw.str == null) { + throw new IllegalArgumentException("op must be non-null"); + } + if (raw.str.toLowerCase(Locale.ROOT).equals(LEGACY_NOOP_STRING)) { + return Op.NOOP; + } + if (validOps.contains(raw.op) == false) { + throw new IllegalArgumentException("unknown op [" + raw.str + "], valid ops are " + String.join(", ", validOps())); + } + return raw.op; + } + + public void setOp(Op op) { + if (validOps.contains(op) == false) { + throw new IllegalArgumentException("unknown op [" + op + "], valid ops are " + String.join(", ", validOps())); + } + store.set(MapBackedMetadata.OP, op == Op.NOOP ? LEGACY_NOOP_STRING : op.name); + } + + public ZonedDateTime getTimestamp() { + return timestamp; + } + + public String getType() { + if (isInsert) { + throw new IllegalStateException("type unavailable for inserts"); + } + return store.getString(TYPE); + } + + public Map getCtx() { + return store.getMap(); + } + + public Map getSource() { + return store.getSource(); + } } } diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java new file mode 100644 index 0000000000000..18b6b6f5fb8af --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.common.util.Maps; + +import java.util.Map; +import java.util.Objects; + +/** Metadata storage backed by a Map for compatibility with the ctx Map used by ingest and update scripts. */ +public class MapBackedMetadata { + public static final String INDEX = "_index"; + public static final String ID = "_id"; + public static final String VERSION = "_version"; + public static final String ROUTING = "_routing"; + public static final String OP = "op"; + public static final String SOURCE = "_source"; + + protected Map map; + + public MapBackedMetadata(Map map) { + this.map = map; + } + + public MapBackedMetadata(int size) { + this.map = Maps.newMapWithExpectedSize(size); + } + + public String getIndex() { + return getString(INDEX); + } + + public MapBackedMetadata setIndex(String index) { + return set(INDEX, index); + } + + public String getId() { + return getString(ID); + } + + public MapBackedMetadata setId(String id) { + return set(ID, id); + } + + public Long getVersion() { + Object obj = getRawVersion(); + if (obj == null) { + return null; + } else if (obj instanceof Number number) { + long version = number.longValue(); + if (number.doubleValue() != version) { + // did we round? + throw new IllegalArgumentException( + "version may only be set to an int or a long but was [" + number + "] with type [" + obj.getClass().getName() + "]" + ); + } + return version; + } + throw new IllegalArgumentException( + "version may only be set to an int or a long but was [" + obj + "] with type [" + obj.getClass().getName() + "]" + ); + } + + protected Object getRawVersion() { + return map.get(VERSION); + } + + public MapBackedMetadata setVersion(Long version) { + return set(VERSION, version); + } + + public void removeVersion() { + map.remove(VERSION); + } + + public String getRouting() { + return getString(ROUTING); + } + + public MapBackedMetadata setRouting(String routing) { + return set(ROUTING, routing); + } + + public RawOp getOp() { + String opStr = getString(OP); + return new RawOp(opStr); + } + + public MapBackedMetadata setOp(Op op) { + if (op == null) { + throw new IllegalArgumentException("operation must be non-null"); + } + return set(OP, op.name); + } + + public String getString(String key) { + return Objects.toString(map.get(key), null); + } + + public MapBackedMetadata set(String key, Object value) { + map.put(key, value); + return this; + } + + public Map getMap() { + return map; + } + + public MapBackedMetadata setSource(Map source) { + map.put(SOURCE, source); + return this; + } + + @SuppressWarnings("unchecked") + public Map getSource() { + Object source = map.get(SOURCE); + if (source instanceof Map map) { + return (Map) map; + } else { + throw new IllegalArgumentException("source should be a map, not [" + source + "] with [" + source.getClass().getName() + "]"); + } + } + + /** + * The Op and underlying String for error messages. + * str may be null. op is always non-null. + */ + public static class RawOp { + public final Op op; + public final String str; + + public RawOp(String str) { + this.str = str; + this.op = Op.fromString(str); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/Op.java b/server/src/main/java/org/elasticsearch/script/field/Op.java new file mode 100644 index 0000000000000..d2bcf77ff2215 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/Op.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import java.util.Locale; + +/** + * Write operation for documents. + * + * This is similar to {@link org.elasticsearch.action.DocWriteRequest.OpType} with NOOP and UNKNOWN added. UPDATE + */ +public enum Op { + NOOP("noop"), + INDEX("index"), + DELETE("delete"), + CREATE("create"), + UPDATE("update"), + UNKNOWN("unknown"); + + public final String name; + + Op(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Get the appropriate OP for the given string. Returns UNKNOWN for no matches + * @param op - the string version of op. + */ + public static Op fromString(String op) { + if (op == null) { + return UNKNOWN; + } + return switch (op.toLowerCase(Locale.ROOT)) { + case "noop" -> NOOP; + case "index" -> INDEX; + case "delete" -> DELETE; + case "create" -> CREATE; + case "update" -> UPDATE; + default -> UNKNOWN; + }; + } + + @Override + public String toString() { + return name; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/UpdateMetadata.java b/server/src/main/java/org/elasticsearch/script/field/UpdateMetadata.java new file mode 100644 index 0000000000000..42509598c2d26 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/UpdateMetadata.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.IngestSourceAndMetadata; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +import static org.elasticsearch.ingest.IngestDocument.Metadata; + +public class UpdateSourceAndMetadata extends IngestSourceAndMetadata { + protected static final Map> VALIDATORS; + static { + Map> v = new HashMap<>(IngestSourceAndMetadata.VALIDATORS); + v.put("_now", ) + } + + public UpdateSourceAndMetadata(String index, + String id, + long version, + String routing, + Op op, + long epochMilli, + String type, + Map source) { + super(updateMetadataMap(index, id, version, routing, epochMilli, op, type), source, epochToZDT(epochMilli), VALIDATORS); + } + + protected static Map updateMetadataMap(String index, + String id, + long version, + String routing, + long epochMilli, + Op op, + String type) { + Map metadata = Maps.newHashMapWithExpectedSize(7); + metadata.put(Metadata.INDEX.getFieldName(), index); + metadata.put(Metadata.ID.getFieldName(), id); + metadata.put(Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(Metadata.ROUTING.getFieldName(), routing); + } + metadata.put("_now", epochToZDT(epochMilli)); + metadata.put("_op", op.toString()); + metadata.put(Metadata.TYPE.getFieldName(), type); + return metadata; + } + + protected static ZonedDateTime epochToZDT(long epochMilli) { + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC); + } + + String getIndex(); + String getId(); + String getRouting(); + long getVersion(); + boolean hasVersion(); + String getOp(); + void setOp(String op); + ZonedDateTime getTimestamp(); + String getType(); +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 332233d2b7d66..87fb163fae03c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -142,7 +142,8 @@ public void testToXContent() throws IOException { }, "index-graveyard": { "tombstones": [] - } + }, + "immutable_state":{} }, "routing_table": { "indices": {} @@ -246,7 +247,8 @@ public void testToXContent() throws IOException { }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state":{} } } }"""), XContentHelper.stripWhitespace(Strings.toString(builder))); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 5a93fa16359c0..cee3bbd8bd506 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -286,7 +286,8 @@ public void testToXContent() throws IOException { }, "index-graveyard": { "tombstones": [] - } + }, + "immutable_state" : { } }, "routing_table": { "indices": { @@ -489,7 +490,8 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } }, "routing_table" : { "indices" : { @@ -699,7 +701,8 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } }, "routing_table" : { "indices" : { @@ -840,7 +843,8 @@ public void testToXContentSameTypeName() throws IOException { }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } }, "routing_table" : { "indices" : { } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index c481fabd83de8..275076dc5d870 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -77,6 +77,7 @@ import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_WRITES; import static org.elasticsearch.cluster.coordination.Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION; import static org.elasticsearch.discovery.PeerFinder.DISCOVERY_FIND_PEERS_INTERVAL_SETTING; +import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.monitor.StatusInfo.Status.HEALTHY; import static org.elasticsearch.monitor.StatusInfo.Status.UNHEALTHY; import static org.elasticsearch.test.NodeRoles.nonMasterNode; @@ -2109,6 +2110,61 @@ public void assertMatched() { } } + @TestLogging( + reason = "testing warning of a single-node cluster having discovery seed hosts", + value = "org.elasticsearch.cluster.coordination.Coordinator:WARN" + ) + public void testLogsWarningPeriodicallyIfSingleNodeClusterHasSeedHosts() throws IllegalAccessException { + final long warningDelayMillis; + final Settings settings; + final String fakeSeedHost = buildNewFakeTransportAddress().toString(); + if (randomBoolean()) { + settings = Settings.builder().putList(DISCOVERY_SEED_HOSTS_SETTING.getKey(), fakeSeedHost).build(); + warningDelayMillis = Coordinator.SINGLE_NODE_CLUSTER_SEED_HOSTS_CHECK_INTERVAL_SETTING.get(settings).millis(); + } else { + warningDelayMillis = randomLongBetween(1, 100000); + settings = Settings.builder() + .put(ClusterFormationFailureHelper.DISCOVERY_CLUSTER_FORMATION_WARNING_TIMEOUT_SETTING.getKey(), warningDelayMillis + "ms") + .putList(DISCOVERY_SEED_HOSTS_SETTING.getKey(), fakeSeedHost) + .build(); + } + logger.info("--> emitting warnings every [{}ms]", warningDelayMillis); + + try (Cluster cluster = new Cluster(1, true, settings)) { + cluster.runRandomly(); + cluster.stabilise(); + + for (int i = scaledRandomIntBetween(1, 10); i >= 0; i--) { + final MockLogAppender mockLogAppender = new MockLogAppender(); + try { + mockLogAppender.start(); + Loggers.addAppender(LogManager.getLogger(Coordinator.class), mockLogAppender); + mockLogAppender.addExpectation(new MockLogAppender.LoggingExpectation() { + String loggedClusterUuid; + + @Override + public void match(LogEvent event) { + final String message = event.getMessage().getFormattedMessage(); + assertThat(message, startsWith("This node is a fully-formed single-node cluster with cluster UUID")); + loggedClusterUuid = (String) event.getMessage().getParameters()[0]; + } + + @Override + public void assertMatched() { + final String clusterUuid = cluster.getAnyNode().getLastAppliedClusterState().metadata().clusterUUID(); + assertThat(loggedClusterUuid + " vs " + clusterUuid, clusterUuid, equalTo(clusterUuid)); + } + }); + cluster.runFor(warningDelayMillis + DEFAULT_DELAY_VARIABILITY, "waiting for warning to be emitted"); + mockLogAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(LogManager.getLogger(Coordinator.class), mockLogAppender); + mockLogAppender.stop(); + } + } + } + } + @TestLogging( reason = "testing LagDetector and CoordinatorPublication logging", value = "org.elasticsearch.cluster.coordination.LagDetector:DEBUG," diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadataTests.java new file mode 100644 index 0000000000000..b8b252c3f2965 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ImmutableStateMetadataTests.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +/** + * Tests for the {@link ImmutableStateMetadata}, {@link ImmutableStateErrorMetadata}, {@link ImmutableStateHandlerMetadata} classes + */ +public class ImmutableStateMetadataTests extends ESTestCase { + + private void equalsTest(boolean addHandlers, boolean addErrors) { + final ImmutableStateMetadata meta = createRandom(addHandlers, addErrors); + assertThat(meta, equalTo(ImmutableStateMetadata.builder(meta.namespace(), meta).build())); + final ImmutableStateMetadata.Builder newMeta = ImmutableStateMetadata.builder(meta.namespace(), meta); + newMeta.putHandler(new ImmutableStateHandlerMetadata("1", Collections.emptySet())); + assertThat(newMeta.build(), not(meta)); + } + + public void testEquals() { + equalsTest(true, true); + equalsTest(true, false); + equalsTest(false, true); + equalsTest(false, false); + } + + private void serializationTest(boolean addHandlers, boolean addErrors) throws IOException { + final ImmutableStateMetadata meta = createRandom(addHandlers, addErrors); + final BytesStreamOutput out = new BytesStreamOutput(); + meta.writeTo(out); + assertThat(ImmutableStateMetadata.readFrom(out.bytes().streamInput()), equalTo(meta)); + } + + public void testSerialization() throws IOException { + serializationTest(true, true); + serializationTest(true, false); + serializationTest(false, true); + serializationTest(false, false); + } + + private void xContentTest(boolean addHandlers, boolean addErrors) throws IOException { + final ImmutableStateMetadata meta = createRandom(addHandlers, addErrors); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + meta.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder)); + parser.nextToken(); // the beginning of the object + assertThat(ImmutableStateMetadata.fromXContent(parser), equalTo(meta)); + } + + public void testXContent() throws IOException { + xContentTest(true, true); + xContentTest(false, true); + xContentTest(true, false); + xContentTest(false, false); + } + + private static ImmutableStateMetadata createRandom(boolean addHandlers, boolean addErrors) { + List handlers = randomList( + 0, + 10, + () -> new ImmutableStateHandlerMetadata(randomAlphaOfLength(5), randomSet(1, 5, () -> randomAlphaOfLength(6))) + ); + + List errors = randomList( + 0, + 10, + () -> new ImmutableStateErrorMetadata( + 1L, + randomFrom(ImmutableStateErrorMetadata.ErrorKind.values()), + randomList(1, 5, () -> randomAlphaOfLength(10)) + ) + ); + + ImmutableStateMetadata.Builder builder = ImmutableStateMetadata.builder(randomAlphaOfLength(7)); + if (addHandlers) { + for (var handlerMeta : handlers) { + builder.putHandler(handlerMeta); + } + } + if (addErrors) { + for (var errorMeta : errors) { + builder.errorMetadata(errorMeta); + } + } + + return builder.build(); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index 77ccb5bf93db5..6e651a70f717c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.cluster.metadata.AliasMetadata.newAliasMetadataBuilder; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; @@ -44,6 +45,24 @@ public class ToAndFromJsonMetadataTests extends ESTestCase { public void testSimpleJsonFromAndTo() throws IOException { IndexMetadata idx1 = createFirstBackingIndex("data-stream1").build(); IndexMetadata idx2 = createFirstBackingIndex("data-stream2").build(); + + ImmutableStateHandlerMetadata hmOne = new ImmutableStateHandlerMetadata("one", Set.of("a", "b")); + ImmutableStateHandlerMetadata hmTwo = new ImmutableStateHandlerMetadata("two", Set.of("c", "d")); + + ImmutableStateErrorMetadata emOne = new ImmutableStateErrorMetadata( + 1L, + ImmutableStateErrorMetadata.ErrorKind.VALIDATION, + List.of("Test error 1", "Test error 2") + ); + + ImmutableStateMetadata immutableStateMetadata = ImmutableStateMetadata.builder("namespace_one") + .errorMetadata(emOne) + .putHandler(hmOne) + .putHandler(hmTwo) + .build(); + + ImmutableStateMetadata immutableStateMetadata1 = ImmutableStateMetadata.builder("namespace_two").putHandler(hmTwo).build(); + Metadata metadata = Metadata.builder() .put( IndexTemplateMetadata.builder("foo") @@ -107,6 +126,8 @@ public void testSimpleJsonFromAndTo() throws IOException { .put(idx2, false) .put(DataStreamTestHelper.newInstance("data-stream1", List.of(idx1.getIndex()))) .put(DataStreamTestHelper.newInstance("data-stream2", List.of(idx2.getIndex()))) + .put(immutableStateMetadata) + .put(immutableStateMetadata1) .build(); XContentBuilder builder = JsonXContent.contentBuilder(); @@ -180,6 +201,10 @@ public void testSimpleJsonFromAndTo() throws IOException { assertThat(parsedMetadata.dataStreams().get("data-stream2").getName(), is("data-stream2")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getTimeStampField().getName(), is("@timestamp")); assertThat(parsedMetadata.dataStreams().get("data-stream2").getIndices(), contains(idx2.getIndex())); + + // immutable 'operator' metadata + assertEquals(immutableStateMetadata, parsedMetadata.immutableStateMetadata().get(immutableStateMetadata.namespace())); + assertEquals(immutableStateMetadata1, parsedMetadata.immutableStateMetadata().get(immutableStateMetadata1.namespace())); } private static final String MAPPING_SOURCE1 = """ @@ -246,7 +271,8 @@ public void testToXContentGateway_FlatSettingTrue_ReduceMappingFalse() throws IO }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } } }""".formatted(Version.CURRENT.id, Version.CURRENT.id), Strings.toString(builder)); } @@ -341,7 +367,8 @@ public void testToXContentAPI_SameTypeName() throws IOException { }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } } }""".formatted(Version.CURRENT.id), Strings.toString(builder)); } @@ -405,7 +432,8 @@ public void testToXContentGateway_FlatSettingFalse_ReduceMappingTrue() throws IO }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } } }""".formatted(Version.CURRENT.id, Version.CURRENT.id), Strings.toString(builder)); } @@ -507,7 +535,8 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce }, "index-graveyard" : { "tombstones" : [ ] - } + }, + "immutable_state" : { } } }""".formatted(Version.CURRENT.id, Version.CURRENT.id), Strings.toString(builder)); } @@ -615,6 +644,187 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce }, "index-graveyard" : { "tombstones" : [ ] + }, + "immutable_state" : { } + } + }""".formatted(Version.CURRENT.id, Version.CURRENT.id), Strings.toString(builder)); + } + + public void testToXContentAPIImmutableMetadata() throws IOException { + Map mapParams = new HashMap<>() { + { + put(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_API); + put("flat_settings", "false"); + put("reduce_mappings", "true"); + } + }; + + Metadata metadata = buildMetadata(); + + ImmutableStateHandlerMetadata hmOne = new ImmutableStateHandlerMetadata("one", Set.of("a", "b")); + ImmutableStateHandlerMetadata hmTwo = new ImmutableStateHandlerMetadata("two", Set.of("c", "d")); + ImmutableStateHandlerMetadata hmThree = new ImmutableStateHandlerMetadata("three", Set.of("e", "f")); + + ImmutableStateErrorMetadata emOne = new ImmutableStateErrorMetadata( + 1L, + ImmutableStateErrorMetadata.ErrorKind.VALIDATION, + List.of("Test error 1", "Test error 2") + ); + + ImmutableStateErrorMetadata emTwo = new ImmutableStateErrorMetadata( + 2L, + ImmutableStateErrorMetadata.ErrorKind.TRANSIENT, + List.of("Test error 3", "Test error 4") + ); + + ImmutableStateMetadata omOne = ImmutableStateMetadata.builder("namespace_one") + .errorMetadata(emOne) + .putHandler(hmOne) + .putHandler(hmTwo) + .build(); + + ImmutableStateMetadata omTwo = ImmutableStateMetadata.builder("namespace_two").errorMetadata(emTwo).putHandler(hmThree).build(); + + metadata = Metadata.builder(metadata).put(omOne).put(omTwo).build(); + + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + metadata.toXContent(builder, new ToXContent.MapParams(mapParams)); + builder.endObject(); + + assertEquals(""" + { + "metadata" : { + "cluster_uuid" : "clusterUUID", + "cluster_uuid_committed" : false, + "cluster_coordination" : { + "term" : 1, + "last_committed_config" : [ + "commitedConfigurationNodeId" + ], + "last_accepted_config" : [ + "acceptedConfigurationNodeId" + ], + "voting_config_exclusions" : [ + { + "node_id" : "exlucdedNodeId", + "node_name" : "excludedNodeName" + } + ] + }, + "templates" : { + "template" : { + "order" : 0, + "index_patterns" : [ + "pattern1", + "pattern2" + ], + "settings" : { + "index" : { + "version" : { + "created" : "%s" + } + } + }, + "mappings" : { }, + "aliases" : { } + } + }, + "indices" : { + "index" : { + "version" : 2, + "mapping_version" : 1, + "settings_version" : 1, + "aliases_version" : 1, + "routing_num_shards" : 1, + "state" : "open", + "settings" : { + "index" : { + "number_of_shards" : "1", + "number_of_replicas" : "2", + "version" : { + "created" : "%s" + } + } + }, + "mappings" : { + "type" : { + "type1" : { + "key" : "value" + } + } + }, + "aliases" : [ + "alias" + ], + "primary_terms" : { + "0" : 1 + }, + "in_sync_allocations" : { + "0" : [ + "allocationId" + ] + }, + "rollover_info" : { + "rolloveAlias" : { + "met_conditions" : { }, + "time" : 1 + } + }, + "system" : false, + "timestamp_range" : { + "shards" : [ ] + } + } + }, + "index-graveyard" : { + "tombstones" : [ ] + }, + "immutable_state" : { + "namespace_one" : { + "version" : 0, + "handlers" : { + "one" : { + "keys" : [ + "a", + "b" + ] + }, + "two" : { + "keys" : [ + "c", + "d" + ] + } + }, + "errors" : { + "version" : 1, + "error_kind" : "validation", + "errors" : [ + "Test error 1", + "Test error 2" + ] + } + }, + "namespace_two" : { + "version" : 0, + "handlers" : { + "three" : { + "keys" : [ + "e", + "f" + ] + } + }, + "errors" : { + "version" : 2, + "error_kind" : "transient", + "errors" : [ + "Test error 3", + "Test error 4" + ] + } + } } } }""".formatted(Version.CURRENT.id, Version.CURRENT.id), Strings.toString(builder)); diff --git a/server/src/test/java/org/elasticsearch/operator/OperatorHandlerTests.java b/server/src/test/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerTests.java similarity index 92% rename from server/src/test/java/org/elasticsearch/operator/OperatorHandlerTests.java rename to server/src/test/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerTests.java index da10b8b74bcf9..86eddbaadad17 100644 --- a/server/src/test/java/org/elasticsearch/operator/OperatorHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/immutablestate/ImmutableClusterStateHandlerTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.operator; +package org.elasticsearch.immutablestate; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -23,9 +23,9 @@ import static org.hamcrest.Matchers.containsInAnyOrder; -public class OperatorHandlerTests extends ESTestCase { +public class ImmutableClusterStateHandlerTests extends ESTestCase { public void testValidation() { - OperatorHandler handler = new OperatorHandler<>() { + ImmutableClusterStateHandler handler = new ImmutableClusterStateHandler<>() { @Override public String name() { return "handler"; @@ -61,7 +61,7 @@ public void testAsMapAndFromMap() throws IOException { } }"""; - OperatorHandler persistentHandler = new OperatorHandler<>() { + ImmutableClusterStateHandler persistentHandler = new ImmutableClusterStateHandler<>() { @Override public String name() { return "persistent"; diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index 7a9fc4398582b..e9960f22ff3d1 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -12,14 +12,21 @@ import org.apache.lucene.util.Constants; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.IndexModule; +import org.elasticsearch.plugins.spi.TestService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.compiler.InMemoryJavaCompiler; +import org.elasticsearch.test.jar.JarUtils; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystemException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -27,8 +34,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -613,6 +622,70 @@ public void testThrowingConstructor() { assertThat(e.getCause().getCause(), hasToString(containsString("test constructor failure"))); } + private ClassLoader buildTestProviderPlugin(String name) throws Exception { + Map sources = Map.of("r.FooPlugin", """ + package r; + import org.elasticsearch.plugins.ActionPlugin; + import org.elasticsearch.plugins.Plugin; + public final class FooPlugin extends Plugin implements ActionPlugin { } + """, "r.FooTestService", Strings.format(""" + package r; + import org.elasticsearch.plugins.spi.TestService; + public final class FooTestService implements TestService { + @Override + public String name() { + return "%s"; + } + } + """, name)); + + var classToBytes = InMemoryJavaCompiler.compile(sources); + + Map jarEntries = new HashMap<>(); + jarEntries.put("r/FooPlugin.class", classToBytes.get("r.FooPlugin")); + jarEntries.put("r/FooTestService.class", classToBytes.get("r.FooTestService")); + jarEntries.put("META-INF/services/org.elasticsearch.plugins.spi.TestService", "r.FooTestService".getBytes(StandardCharsets.UTF_8)); + + Path topLevelDir = createTempDir(getTestName()); + Path jar = topLevelDir.resolve(Strings.format("custom_plugin_%s.jar", name)); + JarUtils.createJarWithEntries(jar, jarEntries); + URL[] urls = new URL[] { jar.toUri().toURL() }; + + URLClassLoader loader = URLClassLoader.newInstance(urls, this.getClass().getClassLoader()); + return loader; + } + + public void testLoadServiceProviders() throws Exception { + ClassLoader fakeClassLoader = buildTestProviderPlugin("integer"); + @SuppressWarnings("unchecked") + Class fakePluginClass = (Class) fakeClassLoader.loadClass("r.FooPlugin"); + + ClassLoader fakeClassLoader1 = buildTestProviderPlugin("string"); + @SuppressWarnings("unchecked") + Class fakePluginClass1 = (Class) fakeClassLoader1.loadClass("r.FooPlugin"); + + assertFalse(fakePluginClass.getClassLoader().equals(fakePluginClass1.getClassLoader())); + + getClass().getModule().addUses(TestService.class); + + PluginsService service = newMockPluginsService(List.of(fakePluginClass, fakePluginClass1)); + + List providers = service.loadServiceProviders(TestService.class); + assertEquals(2, providers.size()); + assertThat(providers.stream().map(p -> p.name()).toList(), containsInAnyOrder("string", "integer")); + + service = newMockPluginsService(List.of(fakePluginClass)); + providers = service.loadServiceProviders(TestService.class); + + assertEquals(1, providers.size()); + assertThat(providers.stream().map(p -> p.name()).toList(), containsInAnyOrder("integer")); + + service = newMockPluginsService(new ArrayList<>()); + providers = service.loadServiceProviders(TestService.class); + + assertEquals(0, providers.size()); + } + private static class TestExtensiblePlugin extends Plugin implements ExtensiblePlugin { private List extensions; diff --git a/server/src/test/java/org/elasticsearch/plugins/spi/TestService.java b/server/src/test/java/org/elasticsearch/plugins/spi/TestService.java new file mode 100644 index 0000000000000..1156ebc1f809a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/plugins/spi/TestService.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.plugins.spi; + +public interface TestService { + String name(); +} diff --git a/server/src/test/java/org/elasticsearch/script/UpdateScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/UpdateScriptMetadataTests.java new file mode 100644 index 0000000000000..b42e428a299ba --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/UpdateScriptMetadataTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.script.field.Op; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class UpdateScriptMetadataTests extends ESTestCase { + UpdateScript.Metadata meta = null; + Map ctx = null; + + protected void insert() { + meta = UpdateScript.insert("myIndex", "myId", Op.CREATE, 0L, Collections.emptyMap()); + ctx = meta.getCtx(); + } + + protected void update() { + meta = UpdateScript.update("myIndex", "myId", 5L, "myRouting", Op.INDEX, 0L, "myType", Collections.emptyMap()); + ctx = meta.getCtx(); + } + + public void testSetOp() { + insert(); + ctx.put("op", null); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, meta::getOp); + assertEquals("op must be non-null", err.getMessage()); + + ctx.put("op", "none"); + assertEquals(Op.NOOP, meta.getOp()); + + ctx.put("op", Op.INDEX.name); + err = expectThrows(IllegalArgumentException.class, meta::getOp); + assertEquals("unknown op [index], valid ops are none, noop, create", err.getMessage()); + + update(); + + ctx.put("op", Op.INDEX.name); + assertEquals(Op.INDEX, meta.getOp()); + } + + public void testType() { + update(); + assertEquals("myType", ctx.get("_type")); + assertEquals("myType", meta.getType()); + } + + public void testInsertDisablesGetters() { + insert(); + IllegalStateException err = expectThrows(IllegalStateException.class, meta::getRouting); + assertEquals("routing unavailable for insert", err.getMessage()); + err = expectThrows(IllegalStateException.class, meta::getVersion); + assertEquals("version unavailable for inserts", err.getMessage()); + err = expectThrows(IllegalStateException.class, meta::getType); + assertEquals("type unavailable for inserts", err.getMessage()); + } + + public void testUpdateEnablesGetters() { + update(); + assertEquals("myRouting", meta.getRouting()); + assertEquals(5, meta.getVersion()); + assertEquals("myType", meta.getType()); + } + + public void testInsertOps() { + insert(); + List valid = meta.validOps(); + assertEquals(3, valid.size()); + assertTrue(valid.contains("none")); + assertTrue(valid.contains("noop")); + assertTrue(valid.contains("create")); + } + + public void testInsertSetOp() { + insert(); + meta.setOp(Op.NOOP); + assertEquals(Op.NOOP, meta.getOp()); + meta.setOp(Op.CREATE); + assertEquals(Op.CREATE, meta.getOp()); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> meta.setOp(Op.INDEX)); + assertEquals("unknown op [index], valid ops are none, noop, create", err.getMessage()); + } + + public void testInsertValidOps() { + insert(); + List valid = meta.validOps(); + assertEquals(3, valid.size()); + assertTrue(valid.contains("none")); + assertTrue(valid.contains("noop")); + assertTrue(valid.contains("create")); + } + + public void testUpdateValidOps() { + update(); + List valid = meta.validOps(); + assertEquals(4, valid.size()); + assertTrue(valid.contains("none")); + assertTrue(valid.contains("noop")); + assertTrue(valid.contains("index")); + assertTrue(valid.contains("delete")); + } + + public void testUpdateSetOp() { + update(); + meta.setOp(Op.NOOP); + assertEquals(Op.NOOP, meta.getOp()); + meta.setOp(Op.INDEX); + assertEquals(Op.INDEX, meta.getOp()); + meta.setOp(Op.DELETE); + assertEquals(Op.DELETE, meta.getOp()); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> meta.setOp(Op.CREATE)); + assertEquals("unknown op [create], valid ops are none, noop, index, delete", err.getMessage()); + } +} diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java new file mode 100644 index 0000000000000..c43d30bbfe205 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class MapBackedMetadataTests extends ESTestCase { + public void testString() { + MapBackedMetadata m = new MapBackedMetadata(1); + String key = "myKey"; + String value = "myValue"; + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + + m.set(key, value); + assertEquals(value, m.getString(key)); + assertEquals(value, m.getMap().get(key)); + + m.set(key, 1); + assertEquals("1", m.getString(key)); + assertEquals(1, m.getMap().get(key)); + + m.set(key, null); + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + + m.set(key, value); + assertEquals(value, m.getString(key)); + assertEquals(value, m.getMap().get(key)); + + m.getMap().remove(key); + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + } + + public void testIndex() { + String index = "myIndex"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getIndex()); + assertNull(m.getMap().get("_index")); + + m.setIndex(index); + assertEquals(index, m.getIndex()); + assertEquals(index, m.getMap().get("_index")); + } + + public void testId() { + String id = "myId"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getId()); + assertNull(m.getMap().get("_id")); + + m.setId(id); + assertEquals(id, m.getId()); + assertEquals(id, m.getMap().get("_id")); + } + + public void testRouting() { + String routing = "myRouting"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getRouting()); + assertNull(m.getMap().get("_routing")); + + m.setRouting(routing); + assertEquals(routing, m.getRouting()); + assertEquals(routing, m.getMap().get("_routing")); + } + + public void testVersion() { + long version = 500; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getVersion()); + assertNull(m.getMap().get("_version")); + + m.setVersion(version); + assertEquals(Long.valueOf(version), m.getVersion()); + assertEquals(version, m.getRawVersion()); + assertEquals(version, m.getMap().get("_version")); + + String badVersion = "badVersion"; + m.set("_version", badVersion); + assertEquals(badVersion, m.getRawVersion()); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getVersion); + assertEquals("version may only be set to an int or a long but was [badVersion] with type [java.lang.String]", err.getMessage()); + + double tooBig = Double.MAX_VALUE; + m.set("_version", tooBig); + assertEquals(tooBig, m.getRawVersion()); + err = expectThrows(IllegalArgumentException.class, m::getVersion); + assertEquals("version may only be set to an int or a long but was [" + tooBig + "] with type [java.lang.Double]", err.getMessage()); + + long justRight = (long) 1 << 52; + m.set("_version", (double) justRight); + assertEquals(Long.valueOf(justRight), m.getVersion()); + assertEquals((double) justRight, m.getRawVersion()); + } + + public void testOp() { + MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata.RawOp raw = m.getOp(); + assertEquals(Op.UNKNOWN, raw.op); + assertNull(raw.str); + assertNull(m.getMap().get("op")); + for (Op op : Op.values()) { + m.setOp(op); + raw = m.getOp(); + assertEquals(op, raw.op); + assertEquals(op.name, raw.str); + assertEquals(op.name, m.getMap().get("op")); + } + + // Unknown op strings return raw.op == UNKNOWN with orig str for error purposes + m.set("op", "gibberish"); + raw = m.getOp(); + assertEquals(Op.UNKNOWN, raw.op); + assertEquals("gibberish", raw.str); + + // Non-string objects get stringified + long longOp = (long) 1 << 60; + m.set("op", longOp); + raw = m.getOp(); + assertEquals(Long.valueOf(longOp).toString(), raw.str); + assertEquals(longOp, m.getMap().get("op")); + + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> m.setOp(null)); + assertEquals("operation must be non-null", err.getMessage()); + } + + public void testSource() { + MapBackedMetadata m = new MapBackedMetadata(1); + Map source = new HashMap<>(); + source.put("foo", "bar"); + + m.setSource(source); + assertEquals(source, m.getSource()); + assertEquals(source, m.getMap().get("_source")); + + source.put("baz", "qux"); + assertEquals("qux", m.getSource().get("baz")); + + m.getMap().put("_source", "mySource"); + + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getSource); + assertEquals("source should be a map, not [mySource] with [java.lang.String]", err.getMessage()); + } + + public void testCtx() { + Map ctx = new HashMap<>(); + ctx.put("_index", "myIndex"); + ctx.put("_id", "myId"); + ctx.put("_version", 200); + ctx.put("_routing", "myRouting"); + ctx.put("op", "create"); + Map source = new HashMap<>(); + source.put("foo", "bar"); + ctx.put("_source", source); + + MapBackedMetadata m = new MapBackedMetadata(ctx); + assertEquals("myIndex", m.getIndex()); + assertEquals("myId", m.getId()); + assertEquals(Long.valueOf(200), m.getVersion()); + assertEquals("myRouting", m.getRouting()); + assertEquals(Op.CREATE, m.getOp().op); + assertEquals("bar", m.getSource().get("foo")); + } +} diff --git a/server/src/test/java/org/elasticsearch/script/field/OpTests.java b/server/src/test/java/org/elasticsearch/script/field/OpTests.java new file mode 100644 index 0000000000000..7c2e54f420457 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/field/OpTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class OpTests extends ESTestCase { + + public void testFromString() { + assertEquals(Op.UNKNOWN, Op.fromString(null)); + assertEquals(Op.UNKNOWN, Op.fromString("random")); + } + + public void testEquivalenceWithOpType() { + Set ops = new HashSet<>(List.of(Op.values())); + for (DocWriteRequest.OpType ot : DocWriteRequest.OpType.values()) { + ops.remove(Op.fromString(ot.getLowercase())); + } + assertTrue(ops.remove(Op.NOOP)); + assertTrue(ops.remove(Op.UNKNOWN)); + assertEquals(0, ops.size()); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index 6c78faa65b7d3..60c10f430c911 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -58,7 +58,7 @@ public MockPluginsService(Settings settings, Environment environment, Collection if (logger.isTraceEnabled()) { logger.trace("plugin loaded from classpath [{}]", pluginInfo); } - pluginsLoaded.add(new LoadedPlugin(pluginInfo, plugin)); + pluginsLoaded.add(new LoadedPlugin(pluginInfo, plugin, pluginClass.getClassLoader(), ModuleLayer.boot())); } this.classpathPlugins = List.copyOf(pluginsLoaded); diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index 270a984ff7343..f0c32d8b14a34 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -369,7 +369,7 @@ protected static OptionalInt getRangeEnd(HttpExchange exchange) { return OptionalInt.of(Math.toIntExact(rangeEnd)); } - protected void sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws IOException { + protected int sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws IOException { final int rangeStart = getRangeStart(exchange); assertThat(rangeStart, lessThan(bytes.length)); final OptionalInt rangeEnd = getRangeEnd(exchange); @@ -391,6 +391,7 @@ protected void sendIncompleteContent(HttpExchange exchange, byte[] bytes) throws if (randomBoolean()) { exchange.getResponseBody().flush(); } + return bytesToSend; } /** diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 0b9f7f8972620..3bd50a98a16bc 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -166,11 +166,11 @@ public boolean execute(Map ctx) { }; return context.factoryClazz.cast(factory); } else if (context.instanceClazz.equals(UpdateScript.class)) { - UpdateScript.Factory factory = (parameters, ctx) -> new UpdateScript(parameters, ctx) { + UpdateScript.Factory factory = (parameters, md) -> new UpdateScript(parameters, md) { @Override public void execute() { final Map vars = new HashMap<>(); - vars.put("ctx", ctx); + vars.put("ctx", md.getCtx()); vars.put("params", parameters); vars.putAll(parameters); script.apply(vars); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 9ebd1d60a9658..a15b67d0409ed 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -993,6 +993,10 @@ public static Map randomMap(int minMapSize, int maxMapSize, Supplie return list; } + public static Set randomSet(int minSetSize, int maxSetSize, Supplier valueConstructor) { + return new HashSet<>(randomList(minSetSize, maxSetSize, valueConstructor)); + } + private static final String[] TIME_SUFFIXES = new String[] { "d", "h", "ms", "s", "m", "micros", "nanos" }; public static String randomTimeValue(int lower, int upper, String... suffixes) { diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java index 4260a8667167c..00ff2981af665 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java @@ -40,7 +40,7 @@ public class IlmHealthIndicatorService implements HealthIndicatorService { public static final String HELP_URL = "https://ela.st/fix-ilm"; public static final UserAction ILM_NOT_RUNNING = new UserAction( - new UserAction.Definition("ilm-not-running", "Start ILM using [POST /_ilm/start].", HELP_URL), + new UserAction.Definition("ilm-not-running", "Start Index Lifecycle Management using [POST /_ilm/start].", HELP_URL), null ); @@ -71,7 +71,7 @@ public HealthIndicatorResult calculate(boolean explain) { if (ilmMetadata.getPolicyMetadatas().isEmpty()) { return createIndicator( GREEN, - "No ILM policies configured", + "No Index Lifecycle Management policies configured", createDetails(explain, ilmMetadata), Collections.emptyList(), Collections.emptyList() @@ -85,11 +85,17 @@ public HealthIndicatorResult calculate(boolean explain) { List.of(ImpactArea.DEPLOYMENT_MANAGEMENT) ) ); - return createIndicator(YELLOW, "ILM is not running", createDetails(explain, ilmMetadata), impacts, List.of(ILM_NOT_RUNNING)); + return createIndicator( + YELLOW, + "Index Lifecycle Management is not running", + createDetails(explain, ilmMetadata), + impacts, + List.of(ILM_NOT_RUNNING) + ); } else { return createIndicator( GREEN, - "ILM is running", + "Index Lifecycle Management is running", createDetails(explain, ilmMetadata), Collections.emptyList(), Collections.emptyList() diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java index 7dd7ff169f329..0c8634f703dee 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java @@ -40,7 +40,7 @@ public class SlmHealthIndicatorService implements HealthIndicatorService { public static final String HELP_URL = "https://ela.st/fix-slm"; public static final UserAction SLM_NOT_RUNNING = new UserAction( - new UserAction.Definition("slm-not-running", "Start SLM using [POST /_slm/start].", HELP_URL), + new UserAction.Definition("slm-not-running", "Start Snapshot Lifecycle Management using [POST /_slm/start].", HELP_URL), null ); @@ -71,7 +71,7 @@ public HealthIndicatorResult calculate(boolean explain) { if (slmMetadata.getSnapshotConfigurations().isEmpty()) { return createIndicator( GREEN, - "No SLM policies configured", + "No Snapshot Lifecycle Management policies configured", createDetails(explain, slmMetadata), Collections.emptyList(), Collections.emptyList() @@ -84,11 +84,17 @@ public HealthIndicatorResult calculate(boolean explain) { List.of(ImpactArea.BACKUP) ) ); - return createIndicator(YELLOW, "SLM is not running", createDetails(explain, slmMetadata), impacts, List.of(SLM_NOT_RUNNING)); + return createIndicator( + YELLOW, + "Snapshot Lifecycle Management is not running", + createDetails(explain, slmMetadata), + impacts, + List.of(SLM_NOT_RUNNING) + ); } else { return createIndicator( GREEN, - "SLM is running", + "Snapshot Lifecycle Management is running", createDetails(explain, slmMetadata), Collections.emptyList(), Collections.emptyList() diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java index 60e370a2dfccb..bf0976e9d1cac 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java @@ -48,7 +48,7 @@ public void testIsGreenWhenRunningAndPoliciesConfigured() { NAME, DATA, GREEN, - "ILM is running", + "Index Lifecycle Management is running", null, new SimpleHealthIndicatorDetails(Map.of("ilm_status", RUNNING, "policies", 1)), Collections.emptyList(), @@ -70,7 +70,7 @@ public void testIsYellowWhenNotRunningAndPoliciesConfigured() { NAME, DATA, YELLOW, - "ILM is not running", + "Index Lifecycle Management is not running", IlmHealthIndicatorService.HELP_URL, new SimpleHealthIndicatorDetails(Map.of("ilm_status", status, "policies", 1)), Collections.singletonList( @@ -99,7 +99,7 @@ public void testIsGreenWhenNotRunningAndNoPolicies() { NAME, DATA, GREEN, - "No ILM policies configured", + "No Index Lifecycle Management policies configured", null, new SimpleHealthIndicatorDetails(Map.of("ilm_status", status, "policies", 0)), Collections.emptyList(), @@ -120,7 +120,7 @@ public void testIsGreenWhenNoMetadata() { NAME, DATA, GREEN, - "No ILM policies configured", + "No Index Lifecycle Management policies configured", null, new SimpleHealthIndicatorDetails(Map.of("ilm_status", RUNNING, "policies", 0)), Collections.emptyList(), diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java index c785e0efc83cc..ec4590c49ce01 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java @@ -49,7 +49,7 @@ public void testIsGreenWhenRunningAndPoliciesConfigured() { NAME, SNAPSHOT, GREEN, - "SLM is running", + "Snapshot Lifecycle Management is running", null, new SimpleHealthIndicatorDetails(Map.of("slm_status", RUNNING, "policies", 1)), Collections.emptyList(), @@ -71,7 +71,7 @@ public void testIsYellowWhenNotRunningAndPoliciesConfigured() { NAME, SNAPSHOT, YELLOW, - "SLM is not running", + "Snapshot Lifecycle Management is not running", SlmHealthIndicatorService.HELP_URL, new SimpleHealthIndicatorDetails(Map.of("slm_status", status, "policies", 1)), Collections.singletonList( @@ -99,7 +99,7 @@ public void testIsGreenWhenNotRunningAndNoPolicies() { NAME, SNAPSHOT, GREEN, - "No SLM policies configured", + "No Snapshot Lifecycle Management policies configured", null, new SimpleHealthIndicatorDetails(Map.of("slm_status", status, "policies", 0)), Collections.emptyList(), @@ -120,7 +120,7 @@ public void testIsGreenWhenNoMetadata() { NAME, SNAPSHOT, GREEN, - "No SLM policies configured", + "No Snapshot Lifecycle Management policies configured", null, new SimpleHealthIndicatorDetails(Map.of("slm_status", RUNNING, "policies", 0)), Collections.emptyList(), diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java index fc045b6fc7b16..60ef944d212c0 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java @@ -9,6 +9,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.Strings; @@ -84,6 +85,7 @@ * - repeat */ @SuppressWarnings("removal") +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/88063") public class TransformContinuousIT extends TransformRestTestCase { private List transformTestCases = new ArrayList<>(); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/140_user_profile.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/140_user_profile.yml index 6d396fbf1684d..a8c3015ecbc8a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/140_user_profile.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/140_user_profile.yml @@ -8,7 +8,7 @@ - do: node_selector: - version: " 8.3.0 - " + version: " 8.4.0 - " security.activate_user_profile: body: > { @@ -22,7 +22,7 @@ - do: node_selector: - version: " 8.3.0 - " + version: " 8.4.0 - " security.get_user_profile: uid: "$profile_uid"