Skip to content

Commit 4e41c5f

Browse files
ywelschywelsch
authored
Handle legacy mappings with placeholder fields (#85059)
As part of #81210 we would like to add support for handling legacy (Elasticsearch 5 and 6) mappings in newer Elasticsearch versions. The idea is to import old mappings "as-is" into Elasticsearch 8, and adapt the mapper parsers so that they can handle those old mappings. Only a select subset of the legacy mapping will actually be parsed, and fields that are neither known to newer ES version nor supported for search will be mapped as "placeholder fields", i.e., they are still represented as fields in the system so that they can give proper error messages when queried by a user. Fields that are supported: - field data types that support doc values only fields - normalizer on keyword fields and date formats on date fields are on supported in so far as they behave similarly across versions. In case they are not, these fields are now updateable on legacy indices so that they can be "fixed" by user. - object fields - nested fields in limited form (not supporting nested queries) - add tests / checks in follow-up PR - multi fields - field aliases - metadata fields - runtime fields (auto-import to be added for future versions) 5.x indices with mappings that have multiple mapping types are collapsed together on a best-effort basis before they are imported. Relates #81210
1 parent 230e566 commit 4e41c5f

32 files changed

+3251
-85
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@
1818
import org.elasticsearch.common.settings.IndexScopedSettings;
1919
import org.elasticsearch.common.settings.Settings;
2020
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
21+
import org.elasticsearch.core.Nullable;
2122
import org.elasticsearch.index.IndexSettings;
2223
import org.elasticsearch.index.analysis.AnalyzerScope;
2324
import org.elasticsearch.index.analysis.IndexAnalyzers;
2425
import org.elasticsearch.index.analysis.NamedAnalyzer;
26+
import org.elasticsearch.index.mapper.DocumentMapper;
2527
import org.elasticsearch.index.mapper.MapperRegistry;
2628
import org.elasticsearch.index.mapper.MapperService;
29+
import org.elasticsearch.index.mapper.Mapping;
2730
import org.elasticsearch.index.similarity.SimilarityService;
2831
import org.elasticsearch.script.ScriptCompiler;
2932
import org.elasticsearch.script.ScriptService;
@@ -89,7 +92,7 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version mi
8992
// Next we have to run this otherwise if we try to create IndexSettings
9093
// with broken settings it would fail in checkMappingsCompatibility
9194
newMetadata = archiveBrokenIndexSettings(newMetadata);
92-
checkMappingsCompatibility(newMetadata);
95+
createAndValidateMapping(newMetadata);
9396
return newMetadata;
9497
}
9598

@@ -126,8 +129,10 @@ private static void checkSupportedVersion(IndexMetadata indexMetadata, Version m
126129
* Note that we don't expect users to encounter mapping incompatibilities, since our index compatibility
127130
* policy guarantees we can read mappings from previous compatible index versions. A failure here would
128131
* indicate a compatibility bug (which are unfortunately not that uncommon).
132+
* @return the mapping
129133
*/
130-
private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
134+
@Nullable
135+
public Mapping createAndValidateMapping(IndexMetadata indexMetadata) {
131136
try {
132137

133138
// We cannot instantiate real analysis server or similarity service at this point because the node
@@ -194,6 +199,8 @@ public Set<Entry<String, NamedAnalyzer>> entrySet() {
194199
scriptService
195200
);
196201
mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
202+
DocumentMapper documentMapper = mapperService.documentMapper();
203+
return documentMapper == null ? null : documentMapper.mapping();
197204
}
198205
} catch (Exception ex) {
199206
// Wrap the inner exception so we have the index name in the exception message

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,12 @@ private FieldValues<Boolean> scriptValues() {
142142
}
143143
}
144144

145-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()));
145+
private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
146+
147+
public static final TypeParser PARSER = new TypeParser(
148+
(n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()),
149+
MINIMUM_COMPATIBILITY_VERSION
150+
);
146151

147152
public static final class BooleanFieldType extends TermBasedFieldType {
148153

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88

99
package org.elasticsearch.index.mapper;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
13+
import org.apache.logging.log4j.message.ParameterizedMessage;
1114
import org.apache.lucene.document.LongPoint;
1215
import org.apache.lucene.document.SortedNumericDocValuesField;
1316
import org.apache.lucene.document.StoredField;
@@ -72,6 +75,7 @@
7275
public final class DateFieldMapper extends FieldMapper {
7376

7477
private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
78+
private static final Logger logger = LogManager.getLogger(DateFieldMapper.class);
7579

7680
public static final String CONTENT_TYPE = "date";
7781
public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
@@ -266,7 +270,12 @@ public Builder(
266270
DateFormatter defaultFormat = resolution == Resolution.MILLISECONDS
267271
? DEFAULT_DATE_TIME_FORMATTER
268272
: DEFAULT_DATE_TIME_NANOS_FORMATTER;
269-
this.format = Parameter.stringParam("format", false, m -> toType(m).format, defaultFormat.pattern());
273+
this.format = Parameter.stringParam(
274+
"format",
275+
indexCreatedVersion.isLegacyIndexVersion(),
276+
m -> toType(m).format,
277+
defaultFormat.pattern()
278+
);
270279
if (dateFormatter != null) {
271280
this.format.setValue(dateFormatter.pattern());
272281
this.locale.setValue(dateFormatter.locale());
@@ -277,7 +286,15 @@ private DateFormatter buildFormatter() {
277286
try {
278287
return DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue());
279288
} catch (IllegalArgumentException e) {
280-
throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
289+
if (indexCreatedVersion.isLegacyIndexVersion()) {
290+
logger.warn(
291+
new ParameterizedMessage("Error parsing format [{}] of legacy index, falling back to default", format.getValue()),
292+
e
293+
);
294+
return DateFormatter.forPattern(format.getDefaultValue()).withLocale(locale.getValue());
295+
} else {
296+
throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
297+
}
281298
}
282299
}
283300

@@ -341,6 +358,8 @@ public DateFieldMapper build(MapperBuilderContext context) {
341358
}
342359
}
343360

361+
private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
362+
344363
public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
345364
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
346365
return new Builder(
@@ -351,7 +370,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
351370
ignoreMalformedByDefault,
352371
c.indexVersionCreated()
353372
);
354-
});
373+
}, MINIMUM_COMPATIBILITY_VERSION);
355374

356375
public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
357376
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
@@ -363,7 +382,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
363382
ignoreMalformedByDefault,
364383
c.indexVersionCreated()
365384
);
366-
});
385+
}, MINIMUM_COMPATIBILITY_VERSION);
367386

368387
public static final class DateFieldType extends MappedFieldType {
369388
protected final DateFormatter dateTimeFormatter;

‎server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.index.mapper;
1010

11+
import org.elasticsearch.Version;
1112
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1213
import org.elasticsearch.xcontent.XContentBuilder;
1314

@@ -124,6 +125,11 @@ public Mapper.Builder parse(String name, Map<String, Object> node, MappingParser
124125
}
125126
return builder.path(path);
126127
}
128+
129+
@Override
130+
public boolean supportsVersion(Version indexCreatedVersion) {
131+
return true;
132+
}
127133
}
128134

129135
public static class Builder extends Mapper.Builder {

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,7 +1191,7 @@ public Builder init(FieldMapper initializer) {
11911191
return this;
11921192
}
11931193

1194-
private void merge(FieldMapper in, Conflicts conflicts) {
1194+
protected void merge(FieldMapper in, Conflicts conflicts) {
11951195
for (Parameter<?> param : getParameters()) {
11961196
param.merge(in, conflicts);
11971197
}
@@ -1238,7 +1238,7 @@ protected void addScriptValidation(
12381238
* Writes the current builder parameter values as XContent
12391239
*/
12401240
@Override
1241-
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
1241+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
12421242
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
12431243
for (Parameter<?> parameter : getParameters()) {
12441244
parameter.toXContent(builder, includeDefaults);
@@ -1304,6 +1304,12 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
13041304
parameter = paramsMap.get(propName);
13051305
}
13061306
if (parameter == null) {
1307+
if (parserContext.indexVersionCreated().isLegacyIndexVersion()) {
1308+
// ignore unknown parameters on legacy indices
1309+
handleUnknownParamOnLegacyIndex(propName, propNode);
1310+
iterator.remove();
1311+
continue;
1312+
}
13071313
if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) {
13081314
deprecationLogger.warn(
13091315
DeprecationCategory.API,
@@ -1352,6 +1358,10 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
13521358
validate();
13531359
}
13541360

1361+
protected void handleUnknownParamOnLegacyIndex(String propName, Object propNode) {
1362+
// ignore
1363+
}
1364+
13551365
protected static ContentPath parentPath(String name) {
13561366
int endPos = name.lastIndexOf(".");
13571367
if (endPos == -1) {
@@ -1388,21 +1398,39 @@ public static final class TypeParser implements Mapper.TypeParser {
13881398

13891399
private final BiFunction<String, MappingParserContext, Builder> builderFunction;
13901400
private final BiConsumer<String, MappingParserContext> contextValidator;
1401+
private final Version minimumCompatibilityVersion; // see Mapper.TypeParser#supportsVersion()
13911402

13921403
/**
13931404
* Creates a new TypeParser
13941405
* @param builderFunction a function that produces a Builder from a name and parsercontext
13951406
*/
13961407
public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction) {
1397-
this(builderFunction, (n, c) -> {});
1408+
this(builderFunction, (n, c) -> {}, Version.CURRENT.minimumIndexCompatibilityVersion());
1409+
}
1410+
1411+
/**
1412+
* Variant of {@link #TypeParser(BiFunction)} that allows to defining a minimumCompatibilityVersion to
1413+
* allow parsing mapping definitions of legacy indices (see {@link Mapper.TypeParser#supportsVersion(Version)}).
1414+
*/
1415+
public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction, Version minimumCompatibilityVersion) {
1416+
this(builderFunction, (n, c) -> {}, minimumCompatibilityVersion);
13981417
}
13991418

14001419
public TypeParser(
14011420
BiFunction<String, MappingParserContext, Builder> builderFunction,
14021421
BiConsumer<String, MappingParserContext> contextValidator
1422+
) {
1423+
this(builderFunction, contextValidator, Version.CURRENT.minimumIndexCompatibilityVersion());
1424+
}
1425+
1426+
private TypeParser(
1427+
BiFunction<String, MappingParserContext, Builder> builderFunction,
1428+
BiConsumer<String, MappingParserContext> contextValidator,
1429+
Version minimumCompatibilityVersion
14031430
) {
14041431
this.builderFunction = builderFunction;
14051432
this.contextValidator = contextValidator;
1433+
this.minimumCompatibilityVersion = minimumCompatibilityVersion;
14061434
}
14071435

14081436
@Override
@@ -1412,6 +1440,11 @@ public Builder parse(String name, Map<String, Object> node, MappingParserContext
14121440
builder.parse(name, parserContext, node);
14131441
return builder;
14141442
}
1443+
1444+
@Override
1445+
public boolean supportsVersion(Version indexCreatedVersion) {
1446+
return indexCreatedVersion.onOrAfter(minimumCompatibilityVersion);
1447+
}
14151448
}
14161449

14171450
}

‎server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,11 @@ public FieldMapper build(MapperBuilderContext context) {
162162

163163
}
164164

165+
private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
166+
165167
public static TypeParser PARSER = new TypeParser(
166-
(n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
168+
(n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
169+
MINIMUM_COMPATIBILITY_VERSION
167170
);
168171

169172
private final Builder builder;

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,10 +175,12 @@ public IpFieldMapper build(MapperBuilderContext context) {
175175

176176
}
177177

178+
private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
179+
178180
public static final TypeParser PARSER = new TypeParser((n, c) -> {
179181
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
180182
return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
181-
});
183+
}, MINIMUM_COMPATIBILITY_VERSION);
182184

183185
public static final class IpFieldType extends SimpleMappedFieldType {
184186

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88

99
package org.elasticsearch.index.mapper;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
13+
import org.apache.logging.log4j.message.ParameterizedMessage;
1114
import org.apache.lucene.analysis.TokenStream;
1215
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
1316
import org.apache.lucene.document.Field;
@@ -78,6 +81,8 @@
7881
*/
7982
public final class KeywordFieldMapper extends FieldMapper {
8083

84+
private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);
85+
8186
public static final String CONTENT_TYPE = "keyword";
8287

8388
public static class Defaults {
@@ -131,8 +136,7 @@ public static class Builder extends FieldMapper.Builder {
131136
private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
132137
private final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> toType(m).similarity);
133138

134-
private final Parameter<String> normalizer = Parameter.stringParam("normalizer", false, m -> toType(m).normalizerName, null)
135-
.acceptsNull();
139+
private final Parameter<String> normalizer;
136140

137141
private final Parameter<Boolean> splitQueriesOnWhitespace = Parameter.boolParam(
138142
"split_queries_on_whitespace",
@@ -156,6 +160,12 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler script
156160
this.indexAnalyzers = indexAnalyzers;
157161
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
158162
this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
163+
this.normalizer = Parameter.stringParam(
164+
"normalizer",
165+
indexCreatedVersion.isLegacyIndexVersion(),
166+
m -> toType(m).normalizerName,
167+
null
168+
).acceptsNull();
159169
this.script.precludesParameters(nullValue);
160170
addScriptValidation(script, indexed, hasDocValues);
161171

@@ -245,7 +255,17 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
245255
assert indexAnalyzers != null;
246256
normalizer = indexAnalyzers.getNormalizer(normalizerName);
247257
if (normalizer == null) {
248-
throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
258+
if (indexCreatedVersion.isLegacyIndexVersion()) {
259+
logger.warn(
260+
new ParameterizedMessage(
261+
"Could not find normalizer [{}] of legacy index, falling back to default",
262+
normalizerName
263+
)
264+
);
265+
normalizer = Lucene.KEYWORD_ANALYZER;
266+
} else {
267+
throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
268+
}
249269
}
250270
searchAnalyzer = quoteAnalyzer = normalizer;
251271
if (splitQueriesOnWhitespace.getValue()) {
@@ -274,8 +294,11 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
274294
}
275295
}
276296

297+
private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
298+
277299
public static final TypeParser PARSER = new TypeParser(
278-
(n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated())
300+
(n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated()),
301+
MINIMUM_COMPATIBILITY_VERSION
279302
);
280303

281304
public static final class KeywordFieldType extends StringFieldType {

‎server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.index.mapper;
1010

11+
import org.elasticsearch.Version;
1112
import org.elasticsearch.common.Strings;
1213
import org.elasticsearch.xcontent.ToXContentFragment;
1314

@@ -34,6 +35,13 @@ public String name() {
3435

3536
public interface TypeParser {
3637
Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext) throws MapperParsingException;
38+
39+
/**
40+
* Whether we can parse this type on indices with the given index created version.
41+
*/
42+
default boolean supportsVersion(Version indexCreatedVersion) {
43+
return indexCreatedVersion.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion());
44+
}
3745
}
3846

3947
private final String simpleName;

0 commit comments

Comments
 (0)