Add Parquet decryption support for Hive tables

lib/trino-parquet/pom.xml

@@ -13,12 +13,25 @@
     <description>Trino - Parquet file format support</description>
 
     <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.google.errorprone</groupId>
             <artifactId>error_prone_annotations</artifactId>
             <optional>true</optional>
         </dependency>
-
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
@@ -29,6 +42,11 @@
             <artifactId>aircompressor-v3</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>json</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>log</artifactId>
@@ -95,6 +113,12 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-filesystem</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.trino</groupId>
             <artifactId>trino-spi</artifactId>

lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java

@@ -21,12 +21,14 @@ public abstract sealed class DataPage
 {
     protected final int valueCount;
     private final OptionalLong firstRowIndex;
+    private final int pageIndex;
 
-    public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex)
+    public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex, int pageIndex)
     {
         super(uncompressedSize);
         this.valueCount = valueCount;
         this.firstRowIndex = firstRowIndex;
+        this.pageIndex = pageIndex;
     }
 
     /**
@@ -41,4 +43,9 @@ public int getValueCount()
     {
         return valueCount;
     }
+
+    public int getPageIndex()
+    {
+        return pageIndex;
+    }
 }

lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java

@@ -35,15 +35,17 @@ public DataPageV1(
             OptionalLong firstRowIndex,
             ParquetEncoding repetitionLevelEncoding,
             ParquetEncoding definitionLevelEncoding,
-            ParquetEncoding valuesEncoding)
+            ParquetEncoding valuesEncoding,
+            int pageIndex)
     {
-        super(uncompressedSize, valueCount, firstRowIndex);
+        super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
         this.slice = requireNonNull(slice, "slice is null");
         this.repetitionLevelEncoding = repetitionLevelEncoding;
         this.definitionLevelEncoding = definitionLevelEncoding;
         this.valuesEncoding = valuesEncoding;
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java

@@ -44,9 +44,10 @@ public DataPageV2(
             int uncompressedSize,
             OptionalLong firstRowIndex,
             Statistics<?> statistics,
-            boolean isCompressed)
+            boolean isCompressed,
+            int pageIndex)
     {
-        super(uncompressedSize, valueCount, firstRowIndex);
+        super(uncompressedSize, valueCount, firstRowIndex, pageIndex);
         this.rowCount = rowCount;
         this.nullCount = nullCount;
         this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
@@ -82,6 +83,7 @@ public ParquetEncoding getDataEncoding()
         return dataEncoding;
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java

@@ -43,6 +43,7 @@ public DictionaryPage(Slice slice, int uncompressedSize, int dictionarySize, Par
                 encoding);
     }
 
+    @Override
     public Slice getSlice()
     {
         return slice;

lib/trino-parquet/src/main/java/io/trino/parquet/Page.java

@@ -13,6 +13,8 @@
  */
 package io.trino.parquet;
 
+import io.airlift.slice.Slice;
+
 public abstract class Page
 {
     protected final int uncompressedSize;
@@ -26,4 +28,6 @@ public int getUncompressedSize()
     {
         return uncompressedSize;
     }
+
+    public abstract Slice getSlice();
 }

lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderEncryptionOptions.java

@@ -0,0 +1,221 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.parquet;
+
+import io.trino.parquet.crypto.keytools.TrinoKeyToolkit;
+import io.trino.parquet.crypto.keytools.TrinoKmsClient;
+
+public class ParquetReaderEncryptionOptions
+{
+    final String cryptoFactoryClass;
+    final String encryptionKmsClientClass;
+    final String encryptionKmsInstanceId;
+    final String encryptionKmsInstanceUrl;
+    final String encryptionKeyAccessToken;
+    final long encryptionCacheLifetimeSeconds;
+    final boolean uniformEncryption;
+    boolean encryptionParameterChecked;
+    final String failsafeEncryptionKeyId;
+    final String columnKeys;
+    final String footerKeyId;
+    final String[] versionedKeyList;
+    final String keyFile;
+    final String[] keyList;
+    final boolean isEncryptionEnvironmentKeys;
+
+    public ParquetReaderEncryptionOptions()
+    {
+        this.cryptoFactoryClass = null;
+        this.encryptionKmsClientClass = null;
+        this.encryptionKmsInstanceId = null;
+        this.encryptionKmsInstanceUrl = null;
+        this.encryptionKeyAccessToken = TrinoKmsClient.KEY_ACCESS_TOKEN_DEFAULT;
+        this.encryptionCacheLifetimeSeconds = TrinoKeyToolkit.CACHE_LIFETIME_DEFAULT_SECONDS;
+        this.uniformEncryption = false;
+        this.encryptionParameterChecked = false;
+        this.failsafeEncryptionKeyId = null;
+        this.footerKeyId = null;
+        this.columnKeys = null;
+        this.versionedKeyList = null;
+        this.keyFile = null;
+        this.keyList = null;
+        this.isEncryptionEnvironmentKeys = false;
+    }
+
+    public ParquetReaderEncryptionOptions(String cryptoFactoryClass,
+                                          String encryptionKmsClientClass,
+                                          String encryptionKmsInstanceId,
+                                          String encryptionKmsInstanceUrl,
+                                          String encryptionKeyAccessToken,
+                                          long encryptionCacheLifetimeSeconds,
+                                          boolean uniformEncryption,
+                                          boolean encryptionParameterChecked,
+                                          String failsafeEncryptionKeyId,
+                                          String footerKeyId,
+                                          String columnKeys,
+                                          String[] versionedKeyList,
+                                          String keyFile,
+                                          String[] keyList,
+                                          boolean isEncryptionEnvironmentKeys)
+    {
+        this.cryptoFactoryClass = cryptoFactoryClass;
+        this.encryptionKmsClientClass = encryptionKmsClientClass;
+        this.encryptionKmsInstanceId = encryptionKmsInstanceId;
+        this.encryptionKmsInstanceUrl = encryptionKmsInstanceUrl;
+        this.encryptionKeyAccessToken = encryptionKeyAccessToken;
+        this.encryptionCacheLifetimeSeconds = encryptionCacheLifetimeSeconds;
+        this.uniformEncryption = uniformEncryption;
+        this.encryptionParameterChecked = encryptionParameterChecked;
+        this.failsafeEncryptionKeyId = failsafeEncryptionKeyId;
+        this.footerKeyId = footerKeyId;
+        this.columnKeys = columnKeys;
+        this.versionedKeyList = versionedKeyList;
+        this.keyFile = keyFile;
+        this.keyList = keyList;
+        this.isEncryptionEnvironmentKeys = isEncryptionEnvironmentKeys;
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionKmsClientClass(String encryptionKmsClientClass)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withCryptoFactoryClass(String cryptoFactoryClass)
+    {
+        return new ParquetReaderEncryptionOptions(cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionKmsInstanceId(String encryptionKmsInstanceId)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionKmsInstanceUrl(String encryptionKmsInstanceUrl)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionKeyAccessToken(String encryptionKeyAccessToken)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionCacheLifetimeSeconds(Long encryptionCacheLifetimeSeconds)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                this.keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+
+    public ParquetReaderEncryptionOptions withEncryptionKeyFile(String keyFile)
+    {
+        return new ParquetReaderEncryptionOptions(this.cryptoFactoryClass,
+                this.encryptionKmsClientClass,
+                this.encryptionKmsInstanceId,
+                this.encryptionKmsInstanceUrl,
+                this.encryptionKeyAccessToken,
+                this.encryptionCacheLifetimeSeconds,
+                this.uniformEncryption,
+                this.encryptionParameterChecked,
+                this.failsafeEncryptionKeyId,
+                this.footerKeyId,
+                this.columnKeys,
+                this.versionedKeyList,
+                keyFile,
+                this.keyList,
+                this.isEncryptionEnvironmentKeys);
+    }
+}