Skip to content

Commit d2817fa

Browse files
cbueschercbuescher
committed
Trying to add new qa test project for 7x archival indices
1 parent 95ba9c4 commit d2817fa

File tree

8 files changed

+3535
-0
lines changed

8 files changed

+3535
-0
lines changed

‎x-pack/qa/repository-old-versions-7x/build.gradle

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import org.elasticsearch.gradle.Architecture
9+
import org.elasticsearch.gradle.OS
10+
import org.elasticsearch.gradle.Version
11+
import org.elasticsearch.gradle.internal.BwcVersions
12+
import org.elasticsearch.gradle.internal.info.BuildParams
13+
import org.elasticsearch.gradle.internal.test.AntFixture
14+
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
15+
import org.elasticsearch.gradle.transform.UnzipTransform
16+
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
17+
18+
apply plugin: 'elasticsearch.jdk-download'
19+
apply plugin: 'elasticsearch.internal-testclusters'
20+
apply plugin: 'elasticsearch.internal-java-rest-test'
21+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
22+
apply plugin: 'elasticsearch.rest-resources'
23+
24+
jdks {
25+
legacy {
26+
vendor = 'adoptium'
27+
version = '8u302+b08'
28+
platform = OS.current().name().toLowerCase()
29+
architecture = Architecture.current().name().toLowerCase()
30+
}
31+
}
32+
33+
restResources {
34+
restApi {
35+
include '_common', 'search'
36+
}
37+
restTests {
38+
includeCore 'search/390_doc_values_search.yml'
39+
}
40+
}
41+
42+
if (OS.current() == OS.MAC && Architecture.current() == Architecture.AARCH64) {
43+
jdks.legacy.vendor = 'zulu'
44+
jdks.legacy.distributionVersion = '8.56.0.23'
45+
}
46+
47+
if (OS.current() == OS.WINDOWS) {
48+
logger.warn("Disabling repository-old-versions tests because we can't get the pid file on windows")
49+
} else {
50+
int currentMajorVersion = org.elasticsearch.gradle.VersionProperties.elasticsearchVersion.major
51+
for (String versionString : ['7.17.25']) {
52+
Version version = Version.fromString(versionString)
53+
String versionNoDots = version.toString().replace('.', '_')
54+
// String configName = "es${versionNoDots}"
55+
56+
// def config = configurations.create(configName)
57+
// config.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
58+
// dependencies.add(configName, artifact)
59+
60+
String repoLocation = "${buildDir}/cluster/shared/repo/${versionNoDots}"
61+
String clusterName = versionNoDots
62+
63+
def testClusterProvider = testClusters.register(clusterName) {
64+
testDistribution = 'DEFAULT'
65+
numberOfNodes = 2
66+
versions = [project.version, project.version] // to test full cluster restart
67+
68+
setting 'path.repo', repoLocation, IGNORE_VALUE
69+
setting 'xpack.license.self_generated.type', 'trial'
70+
71+
setting 'xpack.security.enabled', 'true'
72+
user username: 'admin', password: 'admin-password', role: 'superuser'
73+
74+
setting 'xpack.searchable.snapshot.shared_cache.size', '16MB'
75+
setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB'
76+
}
77+
78+
def oldCluster = testClusters.register("oldES${versionNoDots}Cluster") {
79+
testDistribution = 'DEFAULT'
80+
numberOfNodes = 1
81+
versions = [versionString]
82+
setting 'path.repo', repoLocation, IGNORE_VALUE
83+
setting 'xpack.license.self_generated.type', 'trial'
84+
setting 'xpack.security.enabled', 'true'
85+
user username: 'admin', password: 'admin-password', role: 'superuser'
86+
setting 'xpack.searchable.snapshot.shared_cache.size', '16MB'
87+
setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB'
88+
}
89+
90+
tasks.register("javaRestTestBeforeRestart#${versionNoDots}", StandaloneRestIntegTestTask) {
91+
useCluster testClusterProvider
92+
useCluster testClusters.named("oldES${versionNoDots}Cluster")
93+
doFirst {
94+
delete(repoLocation)
95+
mkdir(repoLocation)
96+
}
97+
systemProperty 'tests.after_restart', 'false'
98+
}
99+
100+
tasks.register("javaRestTestAfterRestart#${versionNoDots}", StandaloneRestIntegTestTask) {
101+
useCluster testClusterProvider
102+
dependsOn "javaRestTestBeforeRestart#${versionNoDots}"
103+
useCluster testClusters.named("oldES${versionNoDots}Cluster")
104+
systemProperty 'tests.after_restart', 'true'
105+
106+
doFirst {
107+
testClusterProvider.get().goToNextVersion()
108+
}
109+
}
110+
111+
tasks.matching { it.name.startsWith("javaRestTest") && it.name.endsWith(versionNoDots) }.configureEach {
112+
doFirst {
113+
def oldClusterPortProvider = testClusters.named("oldES${versionNoDots}Cluster").map { cluster ->
114+
def local = cluster.allHttpSocketURI.find { it.startsWith("127.0.0.1")}
115+
return local.substring(local.lastIndexOf(':') + 1)
116+
}
117+
it.nonInputProperties.systemProperty "tests.es.port", "${-> oldClusterPortProvider.get()}"
118+
println "oldClusterPortProvider " + oldClusterPortProvider.get()
119+
println "allRemoteAccessPortURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allRemoteAccessPortURI
120+
println "allHttpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().allHttpSocketURI
121+
println "httpSocketURI " + testClusters.named("oldES${versionNoDots}Cluster").get().httpSocketURI
122+
}
123+
it.nonInputProperties.systemProperty "tests.repo.location", repoLocation
124+
it.systemProperty "tests.es.version", version.toString()
125+
126+
/* Use a closure on the string to delay evaluation until right before we
127+
* run the integration tests so that we can be sure that the file is
128+
* ready. */
129+
it.nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusterProvider.get().allHttpSocketURI.join(",")}")
130+
it.nonInputProperties.systemProperty('tests.clustername', "${-> testClusterProvider.get().getName()}")
131+
}
132+
133+
tasks.named("check").configure {
134+
dependsOn "javaRestTestAfterRestart#${versionNoDots}"
135+
}
136+
}
137+
}
138+

‎x-pack/qa/repository-old-versions-7x/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.oldrepos;
9+
10+
import com.carrotsearch.randomizedtesting.RandomizedTest;
11+
import com.carrotsearch.randomizedtesting.annotations.Name;
12+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
13+
14+
import org.apache.http.HttpHost;
15+
import org.elasticsearch.Version;
16+
import org.elasticsearch.client.Request;
17+
import org.elasticsearch.client.RestClient;
18+
import org.elasticsearch.common.Strings;
19+
import org.elasticsearch.common.settings.SecureString;
20+
import org.elasticsearch.common.settings.Settings;
21+
import org.elasticsearch.common.util.concurrent.ThreadContext;
22+
import org.elasticsearch.core.Booleans;
23+
import org.elasticsearch.core.PathUtils;
24+
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
25+
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
26+
import org.elasticsearch.xcontent.XContentBuilder;
27+
import org.elasticsearch.xcontent.XContentFactory;
28+
import org.junit.Before;
29+
30+
import java.io.IOException;
31+
32+
/**
33+
* Tests doc-value-based searches against indices imported from clusters older than N-1.
34+
* We reuse the YAML tests in search/390_doc_values_search.yml but have to do the setup
35+
* manually here as the setup is done on the old cluster for which we have to use the
36+
* low-level REST client instead of the YAML set up that only knows how to talk to
37+
* newer ES versions.
38+
*
39+
* We mimic the setup in search/390_doc_values_search.yml here, but adapt it to work
40+
* against older version clusters.
41+
*/
42+
public class DocValueOnlyFieldsIT extends ESClientYamlSuiteTestCase {
43+
44+
final Version oldVersion = Version.fromString(System.getProperty("tests.es.version"));
45+
static boolean setupDone;
46+
47+
public DocValueOnlyFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
48+
super(testCandidate);
49+
}
50+
51+
@ParametersFactory
52+
public static Iterable<Object[]> parameters() throws Exception {
53+
return ESClientYamlSuiteTestCase.createParameters();
54+
}
55+
56+
@Override
57+
protected boolean preserveClusterUponCompletion() {
58+
return true;
59+
}
60+
61+
@Override
62+
protected Settings restClientSettings() {
63+
String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
64+
return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
65+
}
66+
67+
@Override
68+
protected boolean skipSetupSections() {
69+
// setup in the YAML file is replaced by the method below
70+
return true;
71+
}
72+
73+
@Before
74+
public void setupIndex() throws IOException {
75+
final boolean afterRestart = Booleans.parseBoolean(System.getProperty("tests.after_restart"));
76+
if (afterRestart) {
77+
return;
78+
}
79+
80+
// The following is bit of a hack. While we wish we could make this an @BeforeClass, it does not work because the client() is only
81+
// initialized later, so we do it when running the first test
82+
if (setupDone) {
83+
return;
84+
}
85+
86+
setupDone = true;
87+
88+
String repoLocation = PathUtils.get(System.getProperty("tests.repo.location"))
89+
.resolve(RandomizedTest.getContext().getTargetClass().getName())
90+
.toString();
91+
92+
String indexName = "test";
93+
String repoName = "doc_values_repo";
94+
String snapshotName = "snap";
95+
String[] basicTypes = new String[] {
96+
"byte",
97+
"double",
98+
"float",
99+
"half_float",
100+
"integer",
101+
"long",
102+
"short",
103+
"boolean",
104+
"keyword",
105+
"ip",
106+
"geo_point" }; // date is manually added as it need further configuration
107+
108+
int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port"));
109+
try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
110+
Request createIndex = new Request("PUT", "/" + indexName);
111+
int numberOfShards = randomIntBetween(1, 3);
112+
113+
boolean multiTypes = oldVersion.before(Version.V_7_0_0);
114+
115+
XContentBuilder settingsBuilder = XContentFactory.jsonBuilder()
116+
.startObject()
117+
.startObject("settings")
118+
.field("index.number_of_shards", numberOfShards)
119+
.endObject()
120+
.startObject("mappings");
121+
if (multiTypes) {
122+
settingsBuilder.startObject("doc");
123+
}
124+
settingsBuilder.field("dynamic", false).startObject("properties");
125+
for (String type : basicTypes) {
126+
settingsBuilder.startObject(type).field("type", type).endObject();
127+
}
128+
settingsBuilder.startObject("date").field("type", "date").field("format", "yyyy/MM/dd").endObject();
129+
if (multiTypes) {
130+
settingsBuilder.endObject();
131+
}
132+
settingsBuilder.endObject().endObject().endObject();
133+
134+
createIndex.setJsonEntity(Strings.toString(settingsBuilder));
135+
assertOK(oldEs.performRequest(createIndex));
136+
137+
Request doc1 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "1");
138+
doc1.addParameter("refresh", "true");
139+
XContentBuilder bodyDoc1 = XContentFactory.jsonBuilder()
140+
.startObject()
141+
.field("byte", 1)
142+
.field("double", 1.0)
143+
.field("float", 1.0)
144+
.field("half_float", 1.0)
145+
.field("integer", 1)
146+
.field("long", 1)
147+
.field("short", 1)
148+
.field("date", "2017/01/01")
149+
.field("keyword", "key1")
150+
.field("boolean", false)
151+
.field("ip", "192.168.0.1")
152+
.array("geo_point", 13.5, 34.89)
153+
.endObject();
154+
doc1.setJsonEntity(Strings.toString(bodyDoc1));
155+
assertOK(oldEs.performRequest(doc1));
156+
157+
Request doc2 = new Request("PUT", "/" + indexName + "/" + "doc" + "/" + "2");
158+
doc2.addParameter("refresh", "true");
159+
XContentBuilder bodyDoc2 = XContentFactory.jsonBuilder()
160+
.startObject()
161+
.field("byte", 2)
162+
.field("double", 2.0)
163+
.field("float", 2.0)
164+
.field("half_float", 2.0)
165+
.field("integer", 2)
166+
.field("long", 2)
167+
.field("short", 2)
168+
.field("date", "2017/01/02")
169+
.field("keyword", "key2")
170+
.field("boolean", true)
171+
.field("ip", "192.168.0.2")
172+
.array("geo_point", -63.24, 31.0)
173+
.endObject();
174+
doc2.setJsonEntity(Strings.toString(bodyDoc2));
175+
assertOK(oldEs.performRequest(doc2));
176+
177+
// register repo on old ES and take snapshot
178+
Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
179+
createRepoRequest.setJsonEntity(Strings.format("""
180+
{"type":"fs","settings":{"location":"%s"}}
181+
""", repoLocation));
182+
assertOK(oldEs.performRequest(createRepoRequest));
183+
184+
Request createSnapshotRequest = new Request("PUT", "/_snapshot/" + repoName + "/" + snapshotName);
185+
createSnapshotRequest.addParameter("wait_for_completion", "true");
186+
createSnapshotRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
187+
assertOK(oldEs.performRequest(createSnapshotRequest));
188+
}
189+
190+
// register repo on new ES and restore snapshot
191+
Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName);
192+
createRepoRequest2.setJsonEntity(Strings.format("""
193+
{"type":"fs","settings":{"location":"%s"}}
194+
""", repoLocation));
195+
assertOK(client().performRequest(createRepoRequest2));
196+
197+
final Request createRestoreRequest = new Request("POST", "/_snapshot/" + repoName + "/" + snapshotName + "/_restore");
198+
createRestoreRequest.addParameter("wait_for_completion", "true");
199+
createRestoreRequest.setJsonEntity("{\"indices\":\"" + indexName + "\"}");
200+
assertOK(client().performRequest(createRestoreRequest));
201+
}
202+
}

0 commit comments

Comments
 (0)