Skip to content

Commit ac9f30a

Browse files
ywelschywelsch
authored
Provide access to _type in 5.x indices (#83195)
Allows running queries against _type on 5.x indices as well as returning _type in search results. Relates #81210
1 parent b42ba64 commit ac9f30a

File tree

4 files changed

+151
-13
lines changed

4 files changed

+151
-13
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.common.bytes.BytesReference;
1515
import org.elasticsearch.index.mapper.IdFieldMapper;
1616
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
17+
import org.elasticsearch.index.mapper.LegacyTypeFieldMapper;
1718
import org.elasticsearch.index.mapper.MappedFieldType;
1819
import org.elasticsearch.index.mapper.RoutingFieldMapper;
1920
import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -68,7 +69,9 @@ public Status needsField(FieldInfo fieldInfo) {
6869
}
6970
// support _uid for loading older indices
7071
if ("_uid".equals(fieldInfo.name)) {
71-
return Status.YES;
72+
if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) {
73+
return Status.YES;
74+
}
7275
}
7376
// All these fields are single-valued so we can stop when the set is
7477
// empty
@@ -111,8 +114,9 @@ public void stringField(FieldInfo fieldInfo, String value) {
111114
if ("_uid".equals(fieldInfo.name)) {
112115
// 5.x-only
113116
int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can
114-
// type = value.substring(0, delimiterIndex);
117+
String type = value.substring(0, delimiterIndex);
115118
id = value.substring(delimiterIndex + 1);
119+
addValue(LegacyTypeFieldMapper.NAME, type);
116120
} else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
117121
// only applies to 5.x indices that have single_type = true
118122
id = value;

‎server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.index.mapper;
10+
11+
import org.apache.lucene.document.SortedSetDocValuesField;
12+
import org.apache.lucene.sandbox.search.DocValuesTermsQuery;
13+
import org.apache.lucene.search.Query;
14+
import org.apache.lucene.util.BytesRef;
15+
import org.elasticsearch.common.lucene.Lucene;
16+
import org.elasticsearch.index.query.SearchExecutionContext;
17+
18+
import java.util.Collection;
19+
import java.util.Collections;
20+
21+
/**
22+
* Field mapper to access the legacy _type that existed in Elasticsearch 5
23+
*/
24+
public class LegacyTypeFieldMapper extends MetadataFieldMapper {
25+
26+
public static final String NAME = "_type";
27+
28+
public static final String CONTENT_TYPE = "_type";
29+
30+
private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper();
31+
32+
public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);
33+
34+
protected LegacyTypeFieldMapper() {
35+
super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER);
36+
}
37+
38+
static final class LegacyTypeFieldType extends TermBasedFieldType {
39+
40+
LegacyTypeFieldType() {
41+
super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
42+
}
43+
44+
@Override
45+
public String typeName() {
46+
return CONTENT_TYPE;
47+
}
48+
49+
@Override
50+
public boolean isSearchable() {
51+
// The _type field is always searchable.
52+
return true;
53+
}
54+
55+
@Override
56+
public Query termQuery(Object value, SearchExecutionContext context) {
57+
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
58+
}
59+
60+
@Override
61+
public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
62+
BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
63+
return new DocValuesTermsQuery(name(), bytesRefs);
64+
}
65+
66+
@Override
67+
public Query rangeQuery(
68+
Object lowerTerm,
69+
Object upperTerm,
70+
boolean includeLower,
71+
boolean includeUpper,
72+
SearchExecutionContext context
73+
) {
74+
return SortedSetDocValuesField.newSlowRangeQuery(
75+
name(),
76+
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
77+
upperTerm == null ? null : indexedValueForSearch(upperTerm),
78+
includeLower,
79+
includeUpper
80+
);
81+
}
82+
83+
@Override
84+
public boolean mayExistInIndex(SearchExecutionContext context) {
85+
return true;
86+
}
87+
88+
@Override
89+
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
90+
return new StoredValueFetcher(context.lookup(), NAME);
91+
}
92+
}
93+
94+
@Override
95+
protected String contentType() {
96+
return CONTENT_TYPE;
97+
}
98+
}

‎server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public final class MapperRegistry {
2626
private final Map<String, RuntimeField.Parser> runtimeFieldParsers;
2727
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
2828
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers7x;
29+
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers5x;
2930
private final Function<String, Predicate<String>> fieldFilter;
3031

3132
public MapperRegistry(
@@ -40,6 +41,9 @@ public MapperRegistry(
4041
Map<String, MetadataFieldMapper.TypeParser> metadata7x = new LinkedHashMap<>(metadataMapperParsers);
4142
metadata7x.remove(NestedPathFieldMapper.NAME);
4243
this.metadataMapperParsers7x = metadata7x;
44+
Map<String, MetadataFieldMapper.TypeParser> metadata5x = new LinkedHashMap<>(metadata7x);
45+
metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER);
46+
this.metadataMapperParsers5x = metadata5x;
4347
this.fieldFilter = fieldFilter;
4448
}
4549

@@ -62,8 +66,11 @@ public Map<String, RuntimeField.Parser> getRuntimeFieldParsers() {
6266
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
6367
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
6468
return metadataMapperParsers;
69+
} else if (indexCreatedVersion.major < 6) {
70+
return metadataMapperParsers5x;
71+
} else {
72+
return metadataMapperParsers7x;
6573
}
66-
return metadataMapperParsers7x;
6774
}
6875

6976
/**

‎x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.cluster.metadata.MappingMetadata;
3535
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
3636
import org.elasticsearch.common.Strings;
37+
import org.elasticsearch.common.document.DocumentField;
3738
import org.elasticsearch.common.settings.SecureString;
3839
import org.elasticsearch.common.settings.Settings;
3940
import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -136,7 +137,7 @@ && randomBoolean()) {
136137
String id = "testdoc" + i;
137138
expectedIds.add(id);
138139
// use multiple types for ES versions < 6.0.0
139-
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
140+
String type = getType(oldVersion, id);
140141
Request doc = new Request("PUT", "/test/" + type + "/" + id);
141142
doc.addParameter("refresh", "true");
142143
doc.setJsonEntity(sourceForDoc(i));
@@ -146,7 +147,7 @@ && randomBoolean()) {
146147
for (int i = 0; i < extraDocs; i++) {
147148
String id = randomFrom(expectedIds);
148149
expectedIds.remove(id);
149-
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
150+
String type = getType(oldVersion, id);
150151
Request doc = new Request("DELETE", "/test/" + type + "/" + id);
151152
doc.addParameter("refresh", "true");
152153
oldEs.performRequest(doc);
@@ -267,6 +268,10 @@ && randomBoolean()) {
267268
}
268269
}
269270

271+
private String getType(Version oldVersion, String id) {
272+
return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0);
273+
}
274+
270275
private static String sourceForDoc(int i) {
271276
return "{\"test\":\"test" + i + "\",\"val\":" + i + "}";
272277
}
@@ -337,7 +342,7 @@ private void restoreMountAndVerify(
337342
}
338343

339344
// run a search against the index
340-
assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository);
345+
assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
341346

342347
// mount as full copy searchable snapshot
343348
RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots()
@@ -363,7 +368,7 @@ private void restoreMountAndVerify(
363368
);
364369

365370
// run a search against the index
366-
assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository);
371+
assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
367372

368373
// mount as shared cache searchable snapshot
369374
mountSnapshotResponse = client.searchableSnapshots()
@@ -378,12 +383,18 @@ private void restoreMountAndVerify(
378383
assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards());
379384

380385
// run a search against the index
381-
assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository);
386+
assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
382387
}
383388

384389
@SuppressWarnings("removal")
385-
private void assertDocs(String index, int numDocs, Set<String> expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository)
386-
throws IOException {
390+
private void assertDocs(
391+
String index,
392+
int numDocs,
393+
Set<String> expectedIds,
394+
RestHighLevelClient client,
395+
boolean sourceOnlyRepository,
396+
Version oldVersion
397+
) throws IOException {
387398
// run a search against the index
388399
SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT);
389400
logger.info(searchResponse);
@@ -420,9 +431,9 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
420431
// check that doc values can be accessed by (reverse) sorting on numeric val field
421432
// first add mapping for field (this will be done automatically in the future)
422433
XContentBuilder mappingBuilder = JsonXContent.contentBuilder();
423-
mappingBuilder.startObject().startObject("properties").startObject("val");
424-
mappingBuilder.field("type", "long");
425-
mappingBuilder.endObject().endObject().endObject();
434+
mappingBuilder.startObject().startObject("properties");
435+
mappingBuilder.startObject("val").field("type", "long").endObject();
436+
mappingBuilder.endObject().endObject();
426437
assertTrue(
427438
client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged()
428439
);
@@ -442,6 +453,24 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
442453
expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()),
443454
Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList())
444455
);
456+
457+
if (oldVersion.before(Version.fromString("6.0.0"))) {
458+
// search on _type and check that results contain _type information
459+
String randomType = getType(oldVersion, randomFrom(expectedIds));
460+
long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count();
461+
searchResponse = client.search(
462+
new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))),
463+
RequestOptions.DEFAULT
464+
);
465+
logger.info(searchResponse);
466+
assertEquals(typeCount, searchResponse.getHits().getTotalHits().value);
467+
for (SearchHit hit : searchResponse.getHits().getHits()) {
468+
DocumentField typeField = hit.field("_type");
469+
assertNotNull(typeField);
470+
assertThat(typeField.getValue(), instanceOf(String.class));
471+
assertEquals(randomType, typeField.getValue());
472+
}
473+
}
445474
}
446475
}
447476

0 commit comments

Comments
 (0)