[common] Introduce BitmapFileIndexMetaV2.

docs/content/concepts/spec/fileindex.md

@@ -98,9 +98,71 @@ This class use (64-bits) long hash. Store the num hash function (one integer) an
 
 Define `'file-index.bitmap.columns'`.
 
+Bitmap file index format (V2):
+
+<pre>
+
+Bitmap file index format (V2)
++-------------------------------------------------+-----------------
+｜ version (1 byte) = 2                           ｜
++-------------------------------------------------+
+｜ row count (4 bytes int)                        ｜
++-------------------------------------------------+
+｜ non-null value bitmap number (4 bytes int)     ｜
++-------------------------------------------------+
+｜ has null value (1 byte)                        ｜
++-------------------------------------------------+
+｜ null value offset (4 bytes if has null value)  ｜       HEAD
++-------------------------------------------------+
+｜ null bitmap length (4 bytes if has null value) ｜
++-------------------------------------------------+
+｜ bitmap index block number (4 bytes int)        ｜
++-------------------------------------------------+
+｜ value 1 | offset 1                             ｜
++-------------------------------------------------+
+｜ value 2 | offset 2                             ｜
++-------------------------------------------------+
+｜ ...                                            ｜
++-------------------------------------------------+
+｜ bitmap body offset (4 bytes int)               ｜
++-------------------------------------------------+-----------------
+｜ bitmap index block 1                           ｜
++-------------------------------------------------+
+｜ bitmap index block 2                           ｜  INDEX BLOCKS
++-------------------------------------------------+
+｜ ...                                            ｜
++-------------------------------------------------+-----------------
+｜ serialized bitmap 1                            ｜
++-------------------------------------------------+
+｜ serialized bitmap 2                            ｜
++-------------------------------------------------+  BITMAP BLOCKS
+｜ serialized bitmap 3                            ｜
++-------------------------------------------------+
+｜ ...                                            ｜
++-------------------------------------------------+-----------------
+
+index block format:
++-------------------------------------------------+
+｜ entry number (4 bytes int)                     ｜
++-------------------------------------------------+
+｜ value 1 | offset 1 | length 1                  ｜
++-------------------------------------------------+
+｜ value 2 | offset 2 | length 2                  ｜
++-------------------------------------------------+
+｜ ...                                            ｜
++-------------------------------------------------+
+
+value x:                       var bytes for any data type (as bitmap identifier)
+offset:                        4 bytes int (when it is negative, it represents that there is only one value
+                                 and its position is the inverse of the negative value)
+length:                        4 bytes int
+
+</pre>
+
 Bitmap file index format (V1):
 
 <pre>
+
 Bitmap file index format (V1)
 +-------------------------------------------------+-----------------
 ｜ version (1 byte)                               ｜
@@ -135,7 +197,97 @@ offset:                        4 bytes int (when it is negative, it represents t
                                  and its position is the inverse of the negative value)
 </pre>
 
-Integer are all BIG_ENDIAN.
+Integer are all BIG_ENDIAN. In the paimon version that supports v2, the bitmap index version defaults to v2.
+
+Bitmap only support the following data type:
+
+<table class="table table-bordered">
+    <thead>
+    <tr>
+      <th class="text-left" style="width: 10%">Paimon Data Type</th>
+      <th class="text-left" style="width: 5%">Supported</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr>
+      <td><code>TinyIntType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>SmallIntType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>IntType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>BigIntType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>DateType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>TimeType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>LocalZonedTimestampType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>TimestampType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>CharType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>VarCharType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>StringType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>BooleanType</code></td>
+      <td>true</td>
+    </tr>
+    <tr>
+      <td><code>DecimalType(precision, scale)</code></td>
+      <td>false</td>
+    </tr>
+    <tr>
+      <td><code>FloatType</code></td>
+      <td>Not recommended</td>
+    </tr>
+    <tr>
+      <td><code>DoubleType</code></td>
+      <td>Not recommended</td>
+    </tr>
+    <tr>
+      <td><code>VarBinaryType</code>, <code>BinaryType</code></td>
+      <td>false</td>
+    </tr>
+    <tr>
+      <td><code>RowType</code></td>
+      <td>false</td>
+    </tr>
+    <tr>
+      <td><code>MapType</code></td>
+      <td>false</td>
+    </tr>
+    <tr>
+      <td><code>ArrayType</code></td>
+      <td>false</td>
+    </tr>
+    </tbody>
+</table>
+
 
 ## Index: Bit-Slice Index Bitmap
 

paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/BitmapIndexBenchmark.java

@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.benchmark.bitmap;
+
+import org.apache.paimon.benchmark.Benchmark;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.fileindex.FileIndexReader;
+import org.apache.paimon.fileindex.FileIndexResult;
+import org.apache.paimon.fileindex.FileIndexWriter;
+import org.apache.paimon.fileindex.bitmap.BitmapFileIndex;
+import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.RoaringBitmap32;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.jupiter.api.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+
+/** Benchmark for {@link BitmapFileIndex}. */
+public class BitmapIndexBenchmark {
+
+    public static final int ROW_COUNT = 1000000;
+
+    private static final String prefix = "asdfghjkl";
+
+    @Rule public TemporaryFolder folder = new TemporaryFolder();
+
+    @Test
+    public void testQuery10() throws Exception {
+        testQuery(10);
+    }
+
+    @Test
+    public void testQuery100() throws Exception {
+        testQuery(100);
+    }
+
+    @Test
+    public void testQuery1000() throws Exception {
+        testQuery(1000);
+    }
+
+    @Test
+    public void testQuery10000() throws Exception {
+        testQuery(10000);
+    }
+
+    @Test
+    public void testQuery30000() throws Exception {
+        testQuery(30000);
+    }
+
+    @Test
+    public void testQuery50000() throws Exception {
+        testQuery(50000);
+    }
+
+    @Test
+    public void testQuery80000() throws Exception {
+        testQuery(80000);
+    }
+
+    @Test
+    public void testQuery100000() throws Exception {
+        testQuery(100000);
+    }
+
+    private void testQuery(int approxCardinality) throws Exception {
+
+        RoaringBitmap32 middleBm = new RoaringBitmap32();
+
+        Options writeOptions1 = new Options();
+        writeOptions1.setInteger(BitmapFileIndex.VERSION, BitmapFileIndex.VERSION_1);
+        FileIndexWriter writer1 =
+                new BitmapFileIndex(DataTypes.STRING(), writeOptions1).createWriter();
+
+        Options writeOptions2 = new Options();
+        writeOptions1.setInteger(BitmapFileIndex.VERSION, BitmapFileIndex.VERSION_2);
+        FileIndexWriter writer2 =
+                new BitmapFileIndex(DataTypes.STRING(), writeOptions2).createWriter();
+
+        for (int i = 0; i < ROW_COUNT; i++) {
+            int sid = (int) (Math.random() * approxCardinality);
+            if (sid == approxCardinality / 2) {
+                middleBm.add(i);
+            }
+            writer1.write(BinaryString.fromString(prefix + sid));
+            writer2.write(BinaryString.fromString(prefix + sid));
+        }
+
+        folder.create();
+
+        File file1 = folder.newFile("bitmap-index-v1");
+        File file2 = folder.newFile("bitmap-index-v2");
+        FileUtils.writeByteArrayToFile(file1, writer1.serializedBytes());
+        FileUtils.writeByteArrayToFile(file2, writer2.serializedBytes());
+
+        Benchmark benchmark =
+                new Benchmark(
+                                String.format("bitmap-index-query-benchmark-%d", approxCardinality),
+                                100)
+                        .setNumWarmupIters(1)
+                        .setOutputPerIteration(true);
+
+        benchmark.addCase("formatV1", 10, () -> query(approxCardinality, file1, "false", "false"));
+
+        benchmark.addCase(
+                "formatV1-bitmapByteBuffer",
+                10,
+                () -> query(approxCardinality, file1, "false", "true"));
+
+        benchmark.addCase(
+                "formatV1-bufferedInput",
+                10,
+                () -> query(approxCardinality, file1, "true", "false"));
+
+        benchmark.addCase(
+                "formatV1-bufferedInput-bitmapByteBuffer",
+                10,
+                () -> query(approxCardinality, file1, "true", "true"));
+
+        benchmark.addCase("format-v2", 10, () -> query(approxCardinality, file2, "false", "false"));
+
+        benchmark.addCase(
+                "format-v2-bitmapByteBuffer",
+                10,
+                () -> query(approxCardinality, file2, "false", "true"));
+
+        benchmark.addCase(
+                "format-v2-bufferedInput",
+                10,
+                () -> query(approxCardinality, file2, "true", "false"));
+
+        benchmark.addCase(
+                "format-v2-bufferedInput-bitmapByteBuffer",
+                10,
+                () -> query(approxCardinality, file2, "true", "true"));
+
+        benchmark.run();
+    }
+
+    private static void query(
+            int approxCardinality,
+            File file1,
+            String enableBufferedInput,
+            String enableNextOffsetToSize) {
+        try {
+            FieldRef fieldRef = new FieldRef(0, "", DataTypes.STRING());
+            Options options = new Options();
+            options.set(BitmapFileIndex.ENABLE_BUFFERED_INPUT, enableBufferedInput);
+            options.set(BitmapFileIndex.ENABLE_NEXT_OFFSET_TO_SIZE, enableNextOffsetToSize);
+            LocalFileIO.LocalSeekableInputStream localSeekableInputStream =
+                    new LocalFileIO.LocalSeekableInputStream(file1);
+            FileIndexReader reader =
+                    new BitmapFileIndex(DataTypes.STRING(), options)
+                            .createReader(localSeekableInputStream, 0, 0);
+            FileIndexResult result =
+                    reader.visitEqual(
+                            fieldRef, BinaryString.fromString(prefix + (approxCardinality / 2)));
+            ((BitmapIndexResult) result).get();
+        } catch (FileNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/bitmap/RoaringBitmapBenchmark.java

@@ -25,6 +25,7 @@
 import org.junit.jupiter.api.io.TempDir;
 import org.roaringbitmap.RoaringBitmap;
 
+import java.io.BufferedInputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.File;
@@ -78,6 +79,21 @@ public void testDeserialize() throws Exception {
                     }
                 });
 
+        benchmark.addCase(
+                "deserialize(DataInputStream(BufferedInputStream))",
+                10,
+                () -> {
+                    try {
+                        LocalFileIO.LocalSeekableInputStream seekableStream =
+                                new LocalFileIO.LocalSeekableInputStream(file);
+                        DataInputStream input =
+                                new DataInputStream(new BufferedInputStream(seekableStream));
+                        new RoaringBitmap().deserialize(input);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+
         benchmark.addCase(
                 "deserialize(DataInput, byte[])",
                 10,