Skip to content

Commit b3bdb6d

Browse files
mark-vieiramark-vieira
authored
[8.6] Refactor geoip-fixture to remove Docker requirement (#93488) (#94044)
1 parent 43b4152 commit b3bdb6d

File tree

23 files changed

+193
-326
lines changed

23 files changed

+193
-326
lines changed

‎.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-geoip.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
properties-content: |
1111
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
1212
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
13-
geoip_use_service=true
1413
- shell: |
1514
#!/usr/local/bin/runbld --redirect-stderr
16-
$WORKSPACE/.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest
15+
$WORKSPACE/.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dgeoip_use_service=true

‎modules/ingest-geoip/build.gradle

+7-36
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88

99
import org.apache.tools.ant.taskdefs.condition.Os
1010

11-
apply plugin: 'elasticsearch.legacy-yaml-rest-test'
12-
apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
11+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
12+
apply plugin: 'elasticsearch.yaml-rest-compat-test'
1313
apply plugin: 'elasticsearch.internal-cluster-test'
1414

1515
esplugin {
@@ -35,7 +35,11 @@ dependencies {
3535
implementation('com.maxmind.db:maxmind-db:2.0.0')
3636

3737
testImplementation 'org.elasticsearch:geolite2-databases:20191119'
38-
internalClusterTestImplementation project(path: ":modules:reindex")
38+
internalClusterTestImplementation project(':modules:reindex')
39+
internalClusterTestImplementation project(':test:fixtures:geoip-fixture')
40+
yamlRestTestImplementation project(':test:fixtures:geoip-fixture')
41+
42+
clusterModules project(':modules:reindex')
3943
}
4044

4145
restResources {
@@ -48,28 +52,6 @@ tasks.withType(JavaCompile).configureEach {
4852
options.compilerArgs << '-Xlint:-classfile'
4953
}
5054

51-
def useFixture = providers.environmentVariable("geoip_use_service")
52-
.map { s -> Boolean.parseBoolean(s) == false }
53-
.getOrElse(true)
54-
55-
def fixtureAddress = {
56-
assert useFixture: 'closure should not be used without a fixture'
57-
int ephemeralPort = tasks.getByPath(":test:fixtures:geoip-fixture:postProcessFixture").ext."test.fixtures.geoip-fixture.tcp.80"
58-
assert ephemeralPort > 0
59-
return "http://127.0.0.1:${ephemeralPort}/"
60-
}
61-
62-
if (useFixture) {
63-
apply plugin: 'elasticsearch.test.fixtures'
64-
testFixtures.useFixture(':test:fixtures:geoip-fixture', 'geoip-fixture')
65-
}
66-
67-
tasks.named("internalClusterTest").configure {
68-
if (useFixture) {
69-
nonInputProperties.systemProperty "geoip_endpoint", "${-> fixtureAddress()}"
70-
}
71-
}
72-
7355
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
7456
tasks.named("test").configure {
7557
// Windows cannot cleanup database files properly unless it loads everything on heap.
@@ -89,17 +71,6 @@ tasks.named("dependencyLicenses").configure {
8971
ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
9072
}
9173

92-
testClusters.configureEach {
93-
// Needed for database downloader, uses delete-by-query to cleanup old databases from org.elasticsearch.ingest.geoip database system index
94-
module ':modules:reindex'
95-
// Downloader is enabled by default, but in test clusters in build disabled by default,
96-
// but in this module, the downloader should be enabled by default
97-
systemProperty 'ingest.geoip.downloader.enabled.default', 'true'
98-
if (useFixture) {
99-
setting 'ingest.geoip.downloader.endpoint', { "${-> fixtureAddress()}" }
100-
}
101-
}
102-
10374
tasks.named("yamlRestTestV7CompatTransform").configure { task ->
10475
task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests")
10576
// task.skipTest("lang_mustache/50_multi_search_template/Multi-search template with errors", "xxx")

‎modules/ingest-geoip/qa/build.gradle

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
subprojects {
2+
group = "org.elasticsearch.ingest-geoip.qa"
3+
}

‎modules/ingest-geoip/qa/full-cluster-restart/build.gradle

+7-74
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,13 @@ import org.elasticsearch.gradle.VersionProperties
1111
import org.elasticsearch.gradle.internal.info.BuildParams
1212
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
1313

14-
apply plugin: 'elasticsearch.internal-testclusters'
15-
apply plugin: 'elasticsearch.standalone-rest-test'
16-
apply plugin: 'elasticsearch.internal-test-artifact'
14+
apply plugin: 'elasticsearch.internal-java-rest-test'
1715
apply plugin: 'elasticsearch.bwc-test'
1816

19-
def useFixture = providers.environmentVariable("geoip_use_service")
20-
.map { s -> Boolean.parseBoolean(s) == false }
21-
.getOrElse(true)
2217

23-
def fixtureAddress = {
24-
assert useFixture: 'closure should not be used without a fixture'
25-
int ephemeralPort = tasks.getByPath(":test:fixtures:geoip-fixture:postProcessFixture").ext."test.fixtures.geoip-fixture-restart.tcp.80"
26-
assert ephemeralPort > 0
27-
return "http://127.0.0.1:${ephemeralPort}/"
28-
}
29-
30-
if (useFixture) {
31-
apply plugin: 'elasticsearch.test.fixtures'
32-
testFixtures.useFixture(':test:fixtures:geoip-fixture', 'geoip-fixture-restart')
33-
}
34-
35-
tasks.withType(Test).configureEach {
36-
if (useFixture) {
37-
nonInputProperties.systemProperty "geoip_endpoint", "${-> fixtureAddress()}"
38-
}
18+
dependencies {
19+
javaRestTestImplementation project(':test:fixtures:geoip-fixture')
20+
javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest"))
3921
}
4022

4123
assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8 :
@@ -44,57 +26,8 @@ assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).
4426
// once we are ready to test migrations from 8.x to 9.x, we can set the compatible version to 8.0.0
4527
// see https://github.com/elastic/elasticsearch/pull/93666
4628
BuildParams.bwcVersions.withWireCompatible(v -> v.before("7.0.0")) { bwcVersion, baseName ->
47-
def baseCluster = testClusters.register(baseName) {
48-
testDistribution = "DEFAULT"
49-
if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
50-
// When testing older versions we have to first upgrade to 7.last
51-
versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version]
52-
} else {
53-
versions = [bwcVersion.toString(), project.version]
54-
}
55-
numberOfNodes = 2
56-
// some tests rely on the translog not being flushed
57-
setting 'indices.memory.shard_inactive_time', '60m'
58-
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
59-
setting 'xpack.security.enabled', 'false'
60-
if (useFixture) {
61-
setting 'ingest.geoip.downloader.endpoint', { "${-> fixtureAddress()}" }
62-
}
63-
requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
64-
}
65-
66-
tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
67-
useCluster baseCluster
68-
mustRunAfter("precommit")
69-
doFirst {
70-
delete("${buildDir}/cluster/shared/repo/${baseName}")
71-
}
72-
73-
systemProperty 'tests.is_old_cluster', 'true'
74-
}
75-
76-
tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
77-
useCluster baseCluster
78-
dependsOn "${baseName}#oldClusterTest"
79-
doFirst {
80-
baseCluster.get().goToNextVersion()
81-
if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
82-
// When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version.
83-
baseCluster.get().goToNextVersion()
84-
}
85-
}
86-
systemProperty 'tests.is_old_cluster', 'false'
87-
}
88-
89-
String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
90-
tasks.matching { it.name.startsWith(baseName) && it.name.endsWith("ClusterTest") }.configureEach {
91-
it.systemProperty 'tests.old_cluster_version', oldVersion
92-
it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
93-
it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
94-
it.nonInputProperties.systemProperty('tests.clustername', baseName)
95-
}
96-
97-
tasks.register(bwcTaskName(bwcVersion)) {
98-
dependsOn tasks.named("${baseName}#upgradedClusterTest")
29+
tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
30+
usesBwcDistribution(bwcVersion)
31+
systemProperty("tests.old_cluster_version", bwcVersion)
9932
}
10033
}

‎modules/ingest-geoip/qa/full-cluster-restart/src/test/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java renamed to ‎modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

+39-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,21 @@
77
*/
88
package org.elasticsearch.ingest.geoip;
99

10+
import fixture.geoip.GeoIpHttpFixture;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
1014
import org.apache.http.util.EntityUtils;
1115
import org.elasticsearch.client.Request;
16+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
17+
import org.elasticsearch.test.cluster.FeatureFlag;
18+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
1219
import org.elasticsearch.test.rest.ObjectPath;
13-
import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
20+
import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
21+
import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
22+
import org.junit.ClassRule;
23+
import org.junit.rules.RuleChain;
24+
import org.junit.rules.TestRule;
1425

1526
import java.io.IOException;
1627
import java.util.List;
@@ -20,7 +31,33 @@
2031

2132
import static org.hamcrest.Matchers.contains;
2233

23-
public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
34+
public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
35+
36+
private static final boolean useFixture = Boolean.getBoolean("geoip_use_service") == false;
37+
38+
private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
39+
40+
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
41+
.distribution(DistributionType.DEFAULT)
42+
.version(getOldClusterTestVersion())
43+
.nodes(2)
44+
.setting("indices.memory.shard_inactive_time", "60m")
45+
.setting("xpack.security.enabled", "false")
46+
.settings(s -> useFixture ? Map.of("ingest.geoip.downloader.endpoint", fixture.getAddress()) : Map.of())
47+
.feature(FeatureFlag.TIME_SERIES_MODE)
48+
.build();
49+
50+
@ClassRule
51+
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
52+
53+
public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
54+
super(upgradeStatus);
55+
}
56+
57+
@Override
58+
protected ElasticsearchCluster getUpgradeCluster() {
59+
return cluster;
60+
}
2461

2562
public void testGeoIpSystemFeaturesMigration() throws Exception {
2663
if (isRunningAgainstOldCluster()) {

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java

+13
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,16 @@
88

99
package org.elasticsearch.ingest.geoip;
1010

11+
import fixture.geoip.GeoIpHttpFixture;
12+
1113
import org.elasticsearch.common.settings.Setting;
1214
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.core.Booleans;
1316
import org.elasticsearch.core.TimeValue;
1417
import org.elasticsearch.plugins.Plugin;
1518
import org.elasticsearch.test.ESIntegTestCase;
1619
import org.elasticsearch.test.StreamsUtils;
20+
import org.junit.ClassRule;
1721

1822
import java.io.ByteArrayInputStream;
1923
import java.io.IOException;
@@ -25,6 +29,15 @@
2529
import java.util.List;
2630

2731
public abstract class AbstractGeoIpIT extends ESIntegTestCase {
32+
private static final boolean useFixture = Booleans.parseBoolean(System.getProperty("geoip_use_service", "false")) == false;
33+
34+
@ClassRule
35+
public static final GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);
36+
37+
protected String getEndpoint() {
38+
return useFixture ? fixture.getAddress() : null;
39+
}
40+
2841
@Override
2942
protected Collection<Class<? extends Plugin>> nodePlugins() {
3043
return Arrays.asList(IngestGeoIpPlugin.class, IngestGeoIpSettingsPlugin.class);

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ public class GeoIpDownloaderCliIT extends GeoIpDownloaderIT {
1515
@Override
1616
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
1717
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
18-
if (ENDPOINT != null) {
19-
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT + "cli/overview.json");
18+
if (getEndpoint() != null) {
19+
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint() + "cli/overview.json");
2020
}
2121
return settings.build();
2222
}

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

+6-8
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,6 @@
7878

7979
public class GeoIpDownloaderIT extends AbstractGeoIpIT {
8080

81-
protected static final String ENDPOINT = System.getProperty("geoip_endpoint");
82-
8381
@Override
8482
protected Collection<Class<? extends Plugin>> nodePlugins() {
8583
return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class);
@@ -88,8 +86,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
8886
@Override
8987
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
9088
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
91-
if (ENDPOINT != null) {
92-
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT);
89+
if (getEndpoint() != null) {
90+
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint());
9391
}
9492
return settings.build();
9593
}
@@ -148,7 +146,7 @@ public void cleanUp() throws Exception {
148146

149147
@TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/75221")
150148
public void testInvalidTimestamp() throws Exception {
151-
assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
149+
assumeTrue("only test with fixture to have stable results", getEndpoint() != null);
152150
ClusterUpdateSettingsResponse settingsResponse = client().admin()
153151
.cluster()
154152
.prepareUpdateSettings()
@@ -215,7 +213,7 @@ public void testInvalidTimestamp() throws Exception {
215213
}
216214

217215
public void testUpdatedTimestamp() throws Exception {
218-
assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
216+
assumeTrue("only test with fixture to have stable results", getEndpoint() != null);
219217
testGeoIpDatabasesDownload();
220218
long lastCheck = getGeoIpTaskState().getDatabases().get("GeoLite2-ASN.mmdb").lastCheck();
221219
ClusterUpdateSettingsResponse settingsResponse = client().admin()
@@ -285,7 +283,7 @@ public void testGeoIpDatabasesDownload() throws Exception {
285283

286284
@TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/69972")
287285
public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception {
288-
assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
286+
assumeTrue("only test with fixture to have stable results", getEndpoint() != null);
289287
setupDatabasesInConfigDirectory();
290288
// setup:
291289
putPipeline();
@@ -354,7 +352,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception {
354352

355353
@TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/79074")
356354
public void testStartWithNoDatabases() throws Exception {
357-
assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
355+
assumeTrue("only test with fixture to have stable results", getEndpoint() != null);
358356
putPipeline();
359357

360358
// Behaviour without any databases loaded:

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,6 @@
3939
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, maxNumDataNodes = 1)
4040
public class GeoIpDownloaderStatsIT extends AbstractGeoIpIT {
4141

42-
private static final String ENDPOINT = System.getProperty("geoip_endpoint");
43-
4442
@Override
4543
protected Collection<Class<? extends Plugin>> nodePlugins() {
4644
return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class);
@@ -49,8 +47,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
4947
@Override
5048
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
5149
Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
52-
if (ENDPOINT != null) {
53-
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT);
50+
if (getEndpoint() != null) {
51+
settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), getEndpoint());
5452
}
5553
return settings.build();
5654
}

‎modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java

-2
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424

2525
public class GeoIpDownloaderTaskIT extends AbstractGeoIpIT {
2626

27-
protected static final String ENDPOINT = System.getProperty("geoip_endpoint");
28-
2927
@Override
3028
protected Collection<Class<? extends Plugin>> nodePlugins() {
3129
return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, IngestGeoIpSettingsPlugin.class);

0 commit comments

Comments
 (0)