Provide access to _type in 5.x indices

server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IgnoredFieldMapper;
+import org.elasticsearch.index.mapper.LegacyTypeFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -68,7 +69,9 @@ public Status needsField(FieldInfo fieldInfo) {
         }
         // support _uid for loading older indices
         if ("_uid".equals(fieldInfo.name)) {
-            return Status.YES;
+            if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) {
+                return Status.YES;
+            }
         }
         // All these fields are single-valued so we can stop when the set is
         // empty
@@ -111,8 +114,9 @@ public void stringField(FieldInfo fieldInfo, String value) {
         if ("_uid".equals(fieldInfo.name)) {
             // 5.x-only
             int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can
-            // type = value.substring(0, delimiterIndex);
+            String type = value.substring(0, delimiterIndex);
             id = value.substring(delimiterIndex + 1);
+            addValue(LegacyTypeFieldMapper.NAME, type);
         } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
             // only applies to 5.x indices that have single_type = true
             id = value;

server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.sandbox.search.DocValuesTermsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.index.query.SearchExecutionContext;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Field mapper to access the legacy _type that existed in Elasticsearch 5
+ */
+public class LegacyTypeFieldMapper extends MetadataFieldMapper {
+
+    public static final String NAME = "_type";
+
+    public static final String CONTENT_TYPE = "_type";
+
+    private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper();
+
+    public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);
+
+    protected LegacyTypeFieldMapper() {
+        super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER);
+    }
+
+    public static class Defaults {
+
+        public static final FieldType FIELD_TYPE = new FieldType();
+        public static final FieldType NESTED_FIELD_TYPE;
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setStored(true);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.freeze();
+
+            NESTED_FIELD_TYPE = new FieldType();
+            NESTED_FIELD_TYPE.setTokenized(false);
+            NESTED_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            NESTED_FIELD_TYPE.setStored(false);
+            NESTED_FIELD_TYPE.setOmitNorms(true);
+            NESTED_FIELD_TYPE.freeze();
+        }
+    }
+
+    static final class LegacyTypeFieldType extends TermBasedFieldType {
+
+        LegacyTypeFieldType() {
+            super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public boolean isSearchable() {
+            // The _type field is always searchable.
+            return true;
+        }
+
+        @Override
+        public Query termQuery(Object value, SearchExecutionContext context) {
+            return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
+        }
+
+        @Override
+        public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
+            BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
+            return new DocValuesTermsQuery(name(), bytesRefs);
+        }
+
+        @Override
+        public Query rangeQuery(
+            Object lowerTerm,
+            Object upperTerm,
+            boolean includeLower,
+            boolean includeUpper,
+            SearchExecutionContext context
+        ) {
+            return SortedSetDocValuesField.newSlowRangeQuery(
+                name(),
+                lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                includeLower,
+                includeUpper
+            );
+        }
+
+        @Override
+        public boolean mayExistInIndex(SearchExecutionContext context) {
+            return true;
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+            return new StoredValueFetcher(context.lookup(), NAME);
+        }
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+}

server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java

@@ -26,6 +26,7 @@ public final class MapperRegistry {
     private final Map<String, RuntimeField.Parser> runtimeFieldParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers7x;
+    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers5x;
     private final Function<String, Predicate<String>> fieldFilter;
 
     public MapperRegistry(
@@ -40,6 +41,9 @@ public MapperRegistry(
         Map<String, MetadataFieldMapper.TypeParser> metadata7x = new LinkedHashMap<>(metadataMapperParsers);
         metadata7x.remove(NestedPathFieldMapper.NAME);
         this.metadataMapperParsers7x = metadata7x;
+        Map<String, MetadataFieldMapper.TypeParser> metadata5x = new LinkedHashMap<>(metadata7x);
+        metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER);
+        this.metadataMapperParsers5x = metadata5x;
         this.fieldFilter = fieldFilter;
     }
 
@@ -62,8 +66,11 @@ public Map<String, RuntimeField.Parser> getRuntimeFieldParsers() {
     public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
         if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
             return metadataMapperParsers;
+        } else if (indexCreatedVersion.major < 6) {
+            return metadataMapperParsers5x;
+        } else {
+            return metadataMapperParsers7x;
         }
-        return metadataMapperParsers7x;
     }
 
     /**

x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -136,7 +137,7 @@ && randomBoolean()) {
                     String id = "testdoc" + i;
                     expectedIds.add(id);
                     // use multiple types for ES versions < 6.0.0
-                    String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
+                    String type = getType(oldVersion, id);
                     Request doc = new Request("PUT", "/test/" + type + "/" + id);
                     doc.addParameter("refresh", "true");
                     doc.setJsonEntity(sourceForDoc(i));
@@ -146,7 +147,7 @@ && randomBoolean()) {
                 for (int i = 0; i < extraDocs; i++) {
                     String id = randomFrom(expectedIds);
                     expectedIds.remove(id);
-                    String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
+                    String type = getType(oldVersion, id);
                     Request doc = new Request("DELETE", "/test/" + type + "/" + id);
                     doc.addParameter("refresh", "true");
                     oldEs.performRequest(doc);
@@ -267,6 +268,10 @@ && randomBoolean()) {
         }
     }
 
+    private String getType(Version oldVersion, String id) {
+        return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0);
+    }
+
     private static String sourceForDoc(int i) {
         return "{\"test\":\"test" + i + "\",\"val\":" + i + "}";
     }
@@ -337,7 +342,7 @@ private void restoreMountAndVerify(
         }
 
         // run a search against the index
-        assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository);
+        assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
 
         // mount as full copy searchable snapshot
         RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots()
@@ -363,7 +368,7 @@ private void restoreMountAndVerify(
         );
 
         // run a search against the index
-        assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository);
+        assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
 
         // mount as shared cache searchable snapshot
         mountSnapshotResponse = client.searchableSnapshots()
@@ -378,12 +383,18 @@ private void restoreMountAndVerify(
         assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards());
 
         // run a search against the index
-        assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository);
+        assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
     }
 
     @SuppressWarnings("removal")
-    private void assertDocs(String index, int numDocs, Set<String> expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository)
-        throws IOException {
+    private void assertDocs(
+        String index,
+        int numDocs,
+        Set<String> expectedIds,
+        RestHighLevelClient client,
+        boolean sourceOnlyRepository,
+        Version oldVersion
+    ) throws IOException {
         // run a search against the index
         SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT);
         logger.info(searchResponse);
@@ -420,9 +431,9 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
             // check that doc values can be accessed by (reverse) sorting on numeric val field
             // first add mapping for field (this will be done automatically in the future)
             XContentBuilder mappingBuilder = JsonXContent.contentBuilder();
-            mappingBuilder.startObject().startObject("properties").startObject("val");
-            mappingBuilder.field("type", "long");
-            mappingBuilder.endObject().endObject().endObject();
+            mappingBuilder.startObject().startObject("properties");
+            mappingBuilder.startObject("val").field("type", "long").endObject();
+            mappingBuilder.endObject().endObject();
             assertTrue(
                 client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged()
             );
@@ -442,6 +453,24 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
                 expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()),
                 Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList())
             );
+
+            if (oldVersion.before(Version.fromString("6.0.0"))) {
+                // search on _type and check that results contain _type information
+                String randomType = getType(oldVersion, randomFrom(expectedIds));
+                long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count();
+                searchResponse = client.search(
+                    new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))),
+                    RequestOptions.DEFAULT
+                );
+                logger.info(searchResponse);
+                assertEquals(typeCount, searchResponse.getHits().getTotalHits().value);
+                for (SearchHit hit : searchResponse.getHits().getHits()) {
+                    DocumentField typeField = hit.field("_type");
+                    assertNotNull(typeField);
+                    assertThat(typeField.getValue(), instanceOf(String.class));
+                    assertEquals(randomType, typeField.getValue());
+                }
+            }
         }
     }