Handle legacy mappings with placeholder fields

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -18,12 +18,15 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptService;
@@ -126,8 +129,10 @@ private static void checkSupportedVersion(IndexMetadata indexMetadata, Version m
      * Note that we don't expect users to encounter mapping incompatibilities, since our index compatibility
      * policy guarantees we can read mappings from previous compatible index versions. A failure here would
      * indicate a compatibility bug (which are unfortunately not that uncommon).
+     * @return the mapping
      */
-    private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
+    @Nullable
+    public Mapping checkMappingsCompatibility(IndexMetadata indexMetadata) {
         try {
 
             // We cannot instantiate real analysis server or similarity service at this point because the node
@@ -194,6 +199,8 @@ public Set<Entry<String, NamedAnalyzer>> entrySet() {
                     scriptService
                 );
                 mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
+                DocumentMapper documentMapper = mapperService.documentMapper();
+                return documentMapper == null ? null : documentMapper.mapping();
             }
         } catch (Exception ex) {
             // Wrap the inner exception so we have the index name in the exception message

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -142,7 +142,12 @@ private FieldValues<Boolean> scriptValues() {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()));
+    private static final Version minimumCompatibilityVersion = Version.fromString("5.0.0");
+
+    public static final TypeParser PARSER = new TypeParser(
+        (n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()),
+        minimumCompatibilityVersion
+    );
 
     public static final class BooleanFieldType extends TermBasedFieldType {
 

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -72,6 +75,7 @@
 public final class DateFieldMapper extends FieldMapper {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
+    private static final Logger logger = LogManager.getLogger(DateFieldMapper.class);
 
     public static final String CONTENT_TYPE = "date";
     public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
@@ -266,7 +270,12 @@ public Builder(
             DateFormatter defaultFormat = resolution == Resolution.MILLISECONDS
                 ? DEFAULT_DATE_TIME_FORMATTER
                 : DEFAULT_DATE_TIME_NANOS_FORMATTER;
-            this.format = Parameter.stringParam("format", false, m -> toType(m).format, defaultFormat.pattern());
+            this.format = Parameter.stringParam(
+                "format",
+                indexCreatedVersion.isLegacyIndexVersion(),
+                m -> toType(m).format,
+                defaultFormat.pattern()
+            );
             if (dateFormatter != null) {
                 this.format.setValue(dateFormatter.pattern());
                 this.locale.setValue(dateFormatter.locale());
@@ -277,7 +286,15 @@ private DateFormatter buildFormatter() {
             try {
                 return DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue());
             } catch (IllegalArgumentException e) {
-                throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
+                if (indexCreatedVersion.isLegacyIndexVersion()) {
+                    logger.warn(
+                        new ParameterizedMessage("Error parsing format [{}] of legacy index, falling back to default", format.getValue()),
+                        e
+                    );
+                    return DateFormatter.forPattern(format.getDefaultValue()).withLocale(locale.getValue());
+                } else {
+                    throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
+                }
             }
         }
 
@@ -341,6 +358,8 @@ public DateFieldMapper build(MapperBuilderContext context) {
         }
     }
 
+    private static final Version minimumCompatibilityVersion = Version.fromString("5.0.0");
+
     public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(
@@ -351,7 +370,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    });
+    }, minimumCompatibilityVersion);
 
     public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
@@ -363,7 +382,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    });
+    }, minimumCompatibilityVersion);
 
     public static final class DateFieldType extends MappedFieldType {
         protected final DateFormatter dateTimeFormatter;

server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -124,6 +125,11 @@ public Mapper.Builder parse(String name, Map<String, Object> node, MappingParser
             }
             return builder.path(path);
         }
+
+        @Override
+        public boolean supportsVersion(Version indexCreatedVersion) {
+            return true;
+        }
     }
 
     public static class Builder extends Mapper.Builder {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1191,7 +1191,7 @@ public Builder init(FieldMapper initializer) {
             return this;
         }
 
-        private void merge(FieldMapper in, Conflicts conflicts) {
+        protected void merge(FieldMapper in, Conflicts conflicts) {
             for (Parameter<?> param : getParameters()) {
                 param.merge(in, conflicts);
             }
@@ -1238,7 +1238,7 @@ protected void addScriptValidation(
          * Writes the current builder parameter values as XContent
          */
         @Override
-        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
@@ -1304,6 +1304,12 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
                     parameter = paramsMap.get(propName);
                 }
                 if (parameter == null) {
+                    if (parserContext.indexVersionCreated().isLegacyIndexVersion()) {
+                        // ignore unknown parameters on legacy indices
+                        handleUnknownParam(propName, propNode);
+                        iterator.remove();
+                        continue;
+                    }
                     if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) {
                         deprecationLogger.warn(
                             DeprecationCategory.API,
@@ -1352,6 +1358,10 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
             validate();
         }
 
+        protected void handleUnknownParam(String propName, Object propNode) {
+            // ignore
+        }
+
         protected static ContentPath parentPath(String name) {
             int endPos = name.lastIndexOf(".");
             if (endPos == -1) {
@@ -1388,21 +1398,35 @@ public static final class TypeParser implements Mapper.TypeParser {
 
         private final BiFunction<String, MappingParserContext, Builder> builderFunction;
         private final BiConsumer<String, MappingParserContext> contextValidator;
+        private Version minimumCompatibilityVersion; // see Mapper.TypeParser#supportsVersion()
 
         /**
          * Creates a new TypeParser
          * @param builderFunction a function that produces a Builder from a name and parsercontext
          */
         public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction) {
-            this(builderFunction, (n, c) -> {});
+            this(builderFunction, (n, c) -> {}, Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
+
+        public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction, Version minimumCompatibilityVersion) {
+            this(builderFunction, (n, c) -> {}, minimumCompatibilityVersion);
         }
 
         public TypeParser(
             BiFunction<String, MappingParserContext, Builder> builderFunction,
             BiConsumer<String, MappingParserContext> contextValidator
+        ) {
+            this(builderFunction, contextValidator, Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
+
+        public TypeParser(
+            BiFunction<String, MappingParserContext, Builder> builderFunction,
+            BiConsumer<String, MappingParserContext> contextValidator,
+            Version minimumCompatibilityVersion
         ) {
             this.builderFunction = builderFunction;
             this.contextValidator = contextValidator;
+            this.minimumCompatibilityVersion = minimumCompatibilityVersion;
         }
 
         @Override
@@ -1412,6 +1436,11 @@ public Builder parse(String name, Map<String, Object> node, MappingParserContext
             builder.parse(name, parserContext, node);
             return builder;
         }
+
+        @Override
+        public boolean supportsVersion(Version indexCreatedVersion) {
+            return indexCreatedVersion.onOrAfter(minimumCompatibilityVersion);
+        }
     }
 
 }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -160,8 +160,11 @@ public FieldMapper build(MapperBuilderContext context) {
 
     }
 
+    private static final Version minimumCompatibilityVersion = Version.fromString("5.0.0");
+
     public static TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
+        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
+        minimumCompatibilityVersion
     );
 
     private final Builder builder;

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -175,10 +175,12 @@ public IpFieldMapper build(MapperBuilderContext context) {
 
     }
 
+    private static final Version minimumCompatibilityVersion = Version.fromString("5.0.0");
+
     public static final TypeParser PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
-    });
+    }, minimumCompatibilityVersion);
 
     public static final class IpFieldType extends SimpleMappedFieldType {
 

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
@@ -78,6 +81,8 @@
  */
 public final class KeywordFieldMapper extends FieldMapper {
 
+    private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);
+
     public static final String CONTENT_TYPE = "keyword";
 
     public static class Defaults {
@@ -131,8 +136,7 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
         private final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> toType(m).similarity);
 
-        private final Parameter<String> normalizer = Parameter.stringParam("normalizer", false, m -> toType(m).normalizerName, null)
-            .acceptsNull();
+        private final Parameter<String> normalizer;
 
         private final Parameter<Boolean> splitQueriesOnWhitespace = Parameter.boolParam(
             "split_queries_on_whitespace",
@@ -156,6 +160,12 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler script
             this.indexAnalyzers = indexAnalyzers;
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
+            this.normalizer = Parameter.stringParam(
+                "normalizer",
+                indexCreatedVersion.isLegacyIndexVersion(),
+                m -> toType(m).normalizerName,
+                null
+            ).acceptsNull();
             this.script.precludesParameters(nullValue);
             addScriptValidation(script, indexed, hasDocValues);
 
@@ -245,7 +255,17 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
                 assert indexAnalyzers != null;
                 normalizer = indexAnalyzers.getNormalizer(normalizerName);
                 if (normalizer == null) {
-                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                    if (indexCreatedVersion.isLegacyIndexVersion()) {
+                        logger.warn(
+                            new ParameterizedMessage(
+                                "Could not find normalizer [{}] of legacy index, falling back to default",
+                                normalizerName
+                            )
+                        );
+                        normalizer = Lucene.KEYWORD_ANALYZER;
+                    } else {
+                        throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                    }
                 }
                 searchAnalyzer = quoteAnalyzer = normalizer;
                 if (splitQueriesOnWhitespace.getValue()) {
@@ -274,8 +294,11 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
         }
     }
 
+    private static final Version minimumCompatibilityVersion = Version.fromString("5.0.0");
+
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated())
+        (n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated()),
+        minimumCompatibilityVersion
     );
 
     public static final class KeywordFieldType extends StringFieldType {

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xcontent.ToXContentFragment;
 
@@ -34,6 +35,13 @@ public String name() {
 
     public interface TypeParser {
         Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext) throws MapperParsingException;
+
+        /**
+         * Whether we can parse this type on indices with the given index created version.
+         */
+        default boolean supportsVersion(Version indexCreatedVersion) {
+            return indexCreatedVersion.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
     }
 
     private final String simpleName;