Skip to content

Commit 5707b65

Browse files
ywelschywelsch
authored
Copy old mappings to _meta section (#83041)
For archival indices, where mappings might not be parseable by new ES versions anymore, copy the mapping to the `_meta/legacy_mappings` section. Relates #81210
1 parent 7ac0060 commit 5707b65

File tree

4 files changed

+130
-10
lines changed

4 files changed

+130
-10
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1932,8 +1932,19 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
19321932
}
19331933
builder.settings(settings);
19341934
} else if ("mappings".equals(currentFieldName)) {
1935-
// don't try to parse these for now
1936-
parser.skipChildren();
1935+
MapBuilder<String, Object> mappingSourceBuilder = MapBuilder.<String, Object>newMapBuilder();
1936+
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
1937+
if (token == XContentParser.Token.FIELD_NAME) {
1938+
currentFieldName = parser.currentName();
1939+
} else if (token == XContentParser.Token.START_OBJECT) {
1940+
String mappingType = currentFieldName;
1941+
mappingSourceBuilder.put(mappingType, parser.mapOrdered());
1942+
} else {
1943+
throw new IllegalArgumentException("Unexpected token: " + token);
1944+
}
1945+
}
1946+
Map<String, Object> mapping = mappingSourceBuilder.map();
1947+
handleLegacyMapping(builder, mapping);
19371948
} else if ("in_sync_allocations".equals(currentFieldName)) {
19381949
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
19391950
if (token == XContentParser.Token.FIELD_NAME) {
@@ -1957,8 +1968,18 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
19571968
}
19581969
} else if (token == XContentParser.Token.START_ARRAY) {
19591970
if ("mappings".equals(currentFieldName)) {
1960-
// don't try to parse these for now
1961-
parser.skipChildren();
1971+
MapBuilder<String, Object> mappingSourceBuilder = MapBuilder.<String, Object>newMapBuilder();
1972+
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
1973+
Map<String, Object> mapping;
1974+
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
1975+
CompressedXContent compressedXContent = new CompressedXContent(parser.binaryValue());
1976+
mapping = XContentHelper.convertToMap(compressedXContent.compressedReference(), true).v2();
1977+
} else {
1978+
mapping = parser.mapOrdered();
1979+
}
1980+
mappingSourceBuilder.putAll(mapping);
1981+
}
1982+
handleLegacyMapping(builder, mappingSourceBuilder.map());
19621983
} else {
19631984
parser.skipChildren();
19641985
}
@@ -1982,12 +2003,23 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
19822003
}
19832004
XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser);
19842005

1985-
builder.putMapping(MappingMetadata.EMPTY_MAPPINGS); // just make sure it's not empty so that _source can be read
2006+
if (builder.mapping() == null) {
2007+
builder.putMapping(MappingMetadata.EMPTY_MAPPINGS); // just make sure it's not empty so that _source can be read
2008+
}
19862009

19872010
IndexMetadata indexMetadata = builder.build();
19882011
assert indexMetadata.getCreationVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion());
19892012
return indexMetadata;
19902013
}
2014+
2015+
private static void handleLegacyMapping(Builder builder, Map<String, Object> mapping) {
2016+
if (mapping.size() == 1) {
2017+
String mappingType = mapping.keySet().iterator().next();
2018+
builder.putMapping(new MappingMetadata(mappingType, mapping));
2019+
} else if (mapping.size() > 1) {
2020+
builder.putMapping(new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, mapping));
2021+
}
2022+
}
19912023
}
19922024

19932025
/**

‎server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,16 @@ public Map<String, Object> getSourceAsMap() throws ElasticsearchParseException {
141141
return sourceAsMap();
142142
}
143143

144+
/**
145+
* Converts the serialized compressed form of the mappings into a parsed map.
146+
* In contrast to {@link #sourceAsMap()}, this does not remove the type
147+
*/
148+
@SuppressWarnings("unchecked")
149+
public Map<String, Object> rawSourceAsMap() throws ElasticsearchParseException {
150+
Map<String, Object> mapping = XContentHelper.convertToMap(source.compressedReference(), true).v2();
151+
return mapping;
152+
}
153+
144154
public boolean routingRequired() {
145155
return this.routingRequired;
146156
}

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.cluster.metadata.IndexMetadata;
3333
import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
3434
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
35+
import org.elasticsearch.cluster.metadata.MappingMetadata;
3536
import org.elasticsearch.cluster.metadata.Metadata;
3637
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
3738
import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService;
@@ -78,6 +79,7 @@
7879
import java.util.Collections;
7980
import java.util.HashMap;
8081
import java.util.HashSet;
82+
import java.util.LinkedHashMap;
8183
import java.util.List;
8284
import java.util.Locale;
8385
import java.util.Map;
@@ -1294,6 +1296,11 @@ public ClusterState execute(ClusterState currentState) {
12941296
request.indexSettings(),
12951297
request.ignoreIndexSettings()
12961298
);
1299+
if (snapshotIndexMetadata.getCreationVersion()
1300+
.before(currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion())) {
1301+
// adapt index metadata so that it can be understood by current version
1302+
snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata);
1303+
}
12971304
try {
12981305
snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion);
12991306
} catch (Exception ex) {
@@ -1582,6 +1589,32 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState)
15821589
}
15831590
}
15841591

1592+
private IndexMetadata convertLegacyIndex(IndexMetadata snapshotIndexMetadata) {
1593+
MappingMetadata mappingMetadata = snapshotIndexMetadata.mapping();
1594+
Map<String, Object> loadedMappingSource = mappingMetadata.rawSourceAsMap();
1595+
1596+
// store old mapping under _meta/legacy_mappings
1597+
Map<String, Object> legacyMapping = new LinkedHashMap<>();
1598+
boolean sourceOnlySnapshot = snapshotIndexMetadata.getSettings().getAsBoolean("index.source_only", false);
1599+
if (sourceOnlySnapshot) {
1600+
// actual mapping is under "_meta" (but strip type first)
1601+
Object sourceOnlyMeta = mappingMetadata.sourceAsMap().get("_meta");
1602+
if (sourceOnlyMeta instanceof Map<?, ?> sourceOnlyMetaMap) {
1603+
legacyMapping.put("legacy_mappings", sourceOnlyMetaMap);
1604+
}
1605+
} else {
1606+
legacyMapping.put("legacy_mappings", loadedMappingSource);
1607+
}
1608+
1609+
Map<String, Object> newMappingSource = new LinkedHashMap<>();
1610+
newMappingSource.put("_meta", legacyMapping);
1611+
1612+
Map<String, Object> newMapping = new LinkedHashMap<>();
1613+
newMapping.put(mappingMetadata.type(), newMappingSource);
1614+
// TODO: _routing? Perhaps we don't need to obey any routing here as stuff is read-only anyway and get API will be disabled
1615+
return IndexMetadata.builder(snapshotIndexMetadata).putMapping(new MappingMetadata(mappingMetadata.type(), newMapping)).build();
1616+
}
1617+
15851618
private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) {
15861619
return IndexMetadata.builder(snapshotIndexMetadata)
15871620
.state(IndexMetadata.State.OPEN)

‎x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,14 @@
2525
import org.elasticsearch.client.RestClient;
2626
import org.elasticsearch.client.RestHighLevelClient;
2727
import org.elasticsearch.client.indices.CloseIndexRequest;
28+
import org.elasticsearch.client.indices.GetMappingsRequest;
2829
import org.elasticsearch.client.indices.PutMappingRequest;
2930
import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest;
3031
import org.elasticsearch.cluster.SnapshotsInProgress;
3132
import org.elasticsearch.cluster.health.ClusterHealthStatus;
3233
import org.elasticsearch.cluster.metadata.IndexMetadata;
34+
import org.elasticsearch.cluster.metadata.MappingMetadata;
35+
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
3336
import org.elasticsearch.common.Strings;
3437
import org.elasticsearch.common.settings.SecureString;
3538
import org.elasticsearch.common.settings.Settings;
@@ -59,8 +62,13 @@
5962
import java.util.Set;
6063
import java.util.stream.Collectors;
6164

65+
import static org.hamcrest.Matchers.empty;
6266
import static org.hamcrest.Matchers.greaterThan;
67+
import static org.hamcrest.Matchers.hasKey;
6368
import static org.hamcrest.Matchers.hasSize;
69+
import static org.hamcrest.Matchers.instanceOf;
70+
import static org.hamcrest.Matchers.not;
71+
import static org.hamcrest.Matchers.startsWith;
6472

6573
public class OldRepositoryAccessIT extends ESRestTestCase {
6674
@Override
@@ -127,7 +135,9 @@ && randomBoolean()) {
127135
for (int i = 0; i < numDocs + extraDocs; i++) {
128136
String id = "testdoc" + i;
129137
expectedIds.add(id);
130-
Request doc = new Request("PUT", "/test/doc/" + id);
138+
// use multiple types for ES versions < 6.0.0
139+
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
140+
Request doc = new Request("PUT", "/test/" + type + "/" + id);
131141
doc.addParameter("refresh", "true");
132142
doc.setJsonEntity(sourceForDoc(i));
133143
assertOK(oldEs.performRequest(doc));
@@ -136,7 +146,8 @@ && randomBoolean()) {
136146
for (int i = 0; i < extraDocs; i++) {
137147
String id = randomFrom(expectedIds);
138148
expectedIds.remove(id);
139-
Request doc = new Request("DELETE", "/test/doc/" + id);
149+
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
150+
Request doc = new Request("DELETE", "/test/" + type + "/" + id);
140151
doc.addParameter("refresh", "true");
141152
oldEs.performRequest(doc);
142153
}
@@ -218,7 +229,7 @@ && randomBoolean()) {
218229

219230
if (Build.CURRENT.isSnapshot()) {
220231
// restore / mount and check whether searches work
221-
restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository);
232+
restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository, oldVersion);
222233

223234
// close indices
224235
assertTrue(
@@ -236,7 +247,7 @@ && randomBoolean()) {
236247
);
237248

238249
// restore / mount again
239-
restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository);
250+
restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards, sourceOnlyRepository, oldVersion);
240251
}
241252
} finally {
242253
IOUtils.closeWhileHandlingException(
@@ -266,7 +277,8 @@ private void restoreMountAndVerify(
266277
Set<String> expectedIds,
267278
RestHighLevelClient client,
268279
int numberOfShards,
269-
boolean sourceOnlyRepository
280+
boolean sourceOnlyRepository,
281+
Version oldVersion
270282
) throws IOException {
271283
// restore index
272284
RestoreSnapshotResponse restoreSnapshotResponse = client.snapshot()
@@ -291,6 +303,39 @@ private void restoreMountAndVerify(
291303
.getStatus()
292304
);
293305

306+
MappingMetadata mapping = client.indices()
307+
.getMapping(new GetMappingsRequest().indices("restored_test"), RequestOptions.DEFAULT)
308+
.mappings()
309+
.get("restored_test");
310+
logger.info("mapping for {}: {}", mapping.type(), mapping.source().string());
311+
Map<String, Object> root = mapping.sourceAsMap();
312+
assertThat(root, hasKey("_meta"));
313+
assertThat(root.get("_meta"), instanceOf(Map.class));
314+
@SuppressWarnings("unchecked")
315+
Map<String, Object> meta = (Map<String, Object>) root.get("_meta");
316+
assertThat(meta, hasKey("legacy_mappings"));
317+
assertThat(meta.get("legacy_mappings"), instanceOf(Map.class));
318+
@SuppressWarnings("unchecked")
319+
Map<String, Object> legacyMappings = (Map<String, Object>) meta.get("legacy_mappings");
320+
assertThat(legacyMappings.keySet(), not(empty()));
321+
for (Map.Entry<String, Object> entry : legacyMappings.entrySet()) {
322+
String type = entry.getKey();
323+
assertThat(type, startsWith("doc"));
324+
assertThat(entry.getValue(), instanceOf(Map.class));
325+
@SuppressWarnings("unchecked")
326+
Map<String, Object> legacyMapping = (Map<String, Object>) entry.getValue();
327+
assertThat(legacyMapping, hasKey("properties"));
328+
assertThat(legacyMapping.get("properties"), instanceOf(Map.class));
329+
@SuppressWarnings("unchecked")
330+
Map<String, Object> propertiesMapping = (Map<String, Object>) legacyMapping.get("properties");
331+
assertThat(propertiesMapping, hasKey("val"));
332+
assertThat(propertiesMapping.get("val"), instanceOf(Map.class));
333+
@SuppressWarnings("unchecked")
334+
Map<String, Object> valMapping = (Map<String, Object>) propertiesMapping.get("val");
335+
assertThat(valMapping, hasKey("type"));
336+
assertEquals("long", valMapping.get("type"));
337+
}
338+
294339
// run a search against the index
295340
assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository);
296341

0 commit comments

Comments
 (0)