Add doc values support for ES 5 and ES 6 (#82207)

Adds support for reading doc values formats of ES 5 and 6.

Relates #81210

Author: ywelsch

x-pack/plugin/old-lucene-versions/build.gradle

@@ -13,6 +13,4 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
 }
 
-test.enabled = false
-
 addQaCheckDependencies()

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java

@@ -71,6 +71,8 @@ private static void convertToNewFormat(IndexShard indexShard) {
 
     private static SegmentInfos convertToNewerLuceneVersion(OldSegmentInfos oldSegmentInfos) {
         final SegmentInfos segmentInfos = new SegmentInfos(org.apache.lucene.util.Version.LATEST.major);
+        segmentInfos.version = oldSegmentInfos.version;
+        segmentInfos.counter = oldSegmentInfos.counter;
         segmentInfos.setNextWriteGeneration(oldSegmentInfos.getGeneration() + 1);
         final Map<String, String> map = new HashMap<>(oldSegmentInfos.getUserData());
         if (map.containsKey(Engine.HISTORY_UUID_KEY) == false) {
@@ -85,21 +87,21 @@ private static SegmentInfos convertToNewerLuceneVersion(OldSegmentInfos oldSegme
         if (map.containsKey(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID) == false) {
             map.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1");
         }
-        segmentInfos.setUserData(map, true);
+        segmentInfos.setUserData(map, false);
         for (SegmentCommitInfo infoPerCommit : oldSegmentInfos.asList()) {
             final SegmentInfo newInfo = BWCCodec.wrap(infoPerCommit.info);
-
-            segmentInfos.add(
-                new SegmentCommitInfo(
-                    newInfo,
-                    infoPerCommit.getDelCount(),
-                    infoPerCommit.getSoftDelCount(),
-                    infoPerCommit.getDelGen(),
-                    infoPerCommit.getFieldInfosGen(),
-                    infoPerCommit.getDocValuesGen(),
-                    infoPerCommit.getId()
-                )
+            final SegmentCommitInfo commitInfo = new SegmentCommitInfo(
+                newInfo,
+                infoPerCommit.getDelCount(),
+                infoPerCommit.getSoftDelCount(),
+                infoPerCommit.getDelGen(),
+                infoPerCommit.getFieldInfosGen(),
+                infoPerCommit.getDocValuesGen(),
+                infoPerCommit.getId()
             );
+            commitInfo.setDocValuesUpdatesFiles(infoPerCommit.getDocValuesUpdatesFiles());
+            commitInfo.setFieldInfosFiles(infoPerCommit.getFieldInfosFiles());
+            segmentInfos.add(commitInfo);
         }
         return segmentInfos;
     }

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java

@@ -9,7 +9,6 @@
 
 import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.FieldsProducer;
@@ -20,7 +19,6 @@
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
@@ -31,7 +29,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec;
 
 import java.io.IOException;
@@ -60,11 +57,6 @@ public NormsFormat normsFormat() {
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    public DocValuesFormat docValuesFormat() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public TermVectorsFormat termVectorsFormat() {
         throw new UnsupportedOperationException();
@@ -166,14 +158,10 @@ public void write(Directory directory, SegmentInfo segmentInfo, String segmentSu
         };
     }
 
-    // mark all fields as having no postings, no doc values, and no points.
+    // mark all fields as having no postings, no term vectors, no norms, no payloads, no points, and no vectors.
     private static FieldInfos filterFields(FieldInfos fieldInfos) {
         List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
         for (FieldInfo fieldInfo : fieldInfos) {
-            // omit sequence number field so that it doesn't interfere with peer recovery
-            if (fieldInfo.name.equals(SeqNoFieldMapper.NAME)) {
-                continue;
-            }
             fieldInfoCopy.add(
                 new FieldInfo(
                     fieldInfo.name,
@@ -182,8 +170,8 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) {
                     false,
                     false,
                     IndexOptions.NONE,
-                    DocValuesType.NONE,
-                    -1,
+                    fieldInfo.getDocValuesType(),
+                    fieldInfo.getDocValuesGen(),
                     fieldInfo.attributes(),
                     0,
                     0,

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValues.java

@@ -0,0 +1,42 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.index;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A per-document byte[]
+ *
+ * @deprecated Use {@link BinaryDocValues} instead.
+ */
+@Deprecated
+public abstract class LegacyBinaryDocValues {
+
+    /** Sole constructor. (For invocation by subclass
+     * constructors, typically implicit.) */
+    protected LegacyBinaryDocValues() {}
+
+    /** Lookup the value for document.  The returned {@link BytesRef} may be
+     * re-used across calls to {@link #get(int)} so make sure to
+     * {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
+     * around. */
+    public abstract BytesRef get(int docID);
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/index/LegacyBinaryDocValuesWrapper.java

@@ -0,0 +1,93 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.index;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+
+/**
+ * Wraps a {@link LegacyBinaryDocValues} into a {@link BinaryDocValues}.
+ *
+ * @deprecated Implement {@link BinaryDocValues} directly.
+ */
+@Deprecated
+public final class LegacyBinaryDocValuesWrapper extends BinaryDocValues {
+    private final Bits docsWithField;
+    private final LegacyBinaryDocValues values;
+    private final int maxDoc;
+    private int docID = -1;
+
+    public LegacyBinaryDocValuesWrapper(Bits docsWithField, LegacyBinaryDocValues values) {
+        this.docsWithField = docsWithField;
+        this.values = values;
+        this.maxDoc = docsWithField.length();
+    }
+
+    @Override
+    public int docID() {
+        return docID;
+    }
+
+    @Override
+    public int nextDoc() {
+        docID++;
+        while (docID < maxDoc) {
+            if (docsWithField.get(docID)) {
+                return docID;
+            }
+            docID++;
+        }
+        docID = NO_MORE_DOCS;
+        return NO_MORE_DOCS;
+    }
+
+    @Override
+    public int advance(int target) {
+        if (target < docID) {
+            throw new IllegalArgumentException("cannot advance backwards: docID=" + docID + " target=" + target);
+        }
+        if (target == NO_MORE_DOCS) {
+            this.docID = NO_MORE_DOCS;
+        } else {
+            this.docID = target - 1;
+            nextDoc();
+        }
+        return docID;
+    }
+
+    @Override
+    public boolean advanceExact(int target) throws IOException {
+        docID = target;
+        return docsWithField.get(target);
+    }
+
+    @Override
+    public long cost() {
+        return 0;
+    }
+
+    @Override
+    public BytesRef binaryValue() {
+        return values.get(docID);
+    }
+}