Skip to content

Commit 45f074d

Browse files
j1wonparkj1wonpark
authored
[AMORO-4127] Add mixed Flink 1.19 support (#4141)
* [AMORO-4127][mixed-flink] Add mixed Flink 1.19 support - Add v1.19/amoro-mixed-flink-1.19 and v1.19/amoro-mixed-flink-runtime-1.19 modules - Register new modules in parent pom.xml - Update documentation to include Flink 1.19 in supported version matrix - Add kind cluster config for local E2E testing Signed-off-by: Jiwon Park <jpark92@outlook.kr>
1 parent fe98a6c commit 45f074d

File tree

10 files changed

+355
-8
lines changed

10 files changed

+355
-8
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ Amoro support multiple processing engines for Mixed format as below:
8282

8383
| Processing Engine | Version | Batch Read | Batch Write | Batch Overwrite | Streaming Read | Streaming Write | Create Table | Alter Table |
8484
|-------------------|------------------------|-------------|-------------|-----------------|----------------|-----------------|--------------|-------------|
85-
| Flink | 1.17.x, 1.18.x | &#x2714; | &#x2714; | &#x2716; | &#x2714; | &#x2714; | &#x2714; | &#x2716; |
85+
| Flink | 1.17.x, 1.18.x, 1.19.x | &#x2714; | &#x2714; | &#x2716; | &#x2714; | &#x2714; | &#x2714; | &#x2716; |
8686
| Spark | 3.3, 3.4, 3.5 | &#x2714; | &#x2714; | &#x2714; | &#x2716; | &#x2716; | &#x2714; | &#x2714; |
8787
| Hive | 2.x, 3.x | &#x2714; | &#x2716; | &#x2714; | &#x2716; | &#x2716; | &#x2716; | &#x2714; |
8888
| Trino | 406 | &#x2714; | &#x2716; | &#x2714; | &#x2716; | &#x2716; | &#x2716; | &#x2714; |

‎amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434

3535
<properties>
3636
<assertj.version>3.21.0</assertj.version>
37-
<flink.version>1.18.1</flink.version>
37+
<flink.version>1.19.1</flink.version>
3838
</properties>
3939

4040
<dependencies>
@@ -63,7 +63,7 @@
6363

6464
<dependency>
6565
<groupId>org.apache.iceberg</groupId>
66-
<artifactId>iceberg-flink-1.18</artifactId>
66+
<artifactId>iceberg-flink-1.19</artifactId>
6767
<version>${iceberg.version}</version>
6868
<scope>provided</scope>
6969
<exclusions>
@@ -84,7 +84,7 @@
8484

8585
<dependency>
8686
<groupId>org.apache.paimon</groupId>
87-
<artifactId>paimon-flink-1.18</artifactId>
87+
<artifactId>paimon-flink-1.19</artifactId>
8888
<version>${paimon.version}</version>
8989
<scope>provided</scope>
9090
</dependency>
@@ -219,7 +219,7 @@
219219

220220
<dependency>
221221
<groupId>org.apache.iceberg</groupId>
222-
<artifactId>iceberg-flink-1.18</artifactId>
222+
<artifactId>iceberg-flink-1.19</artifactId>
223223
<version>${iceberg.version}</version>
224224
<classifier>tests</classifier>
225225
<scope>test</scope>

‎amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/main/java/org/apache/amoro/flink/write/MixedFormatWriter.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.flink.streaming.api.operators.Output;
3333
import org.apache.flink.streaming.api.watermark.Watermark;
3434
import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
35+
import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes;
3536
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
3637
import org.apache.flink.streaming.runtime.tasks.StreamTask;
3738
import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
@@ -212,6 +213,9 @@ public void collect(StreamRecord<RowData> rowDataStreamRecord) {}
212213
@Override
213214
public void emitLatencyMarker(LatencyMarker latencyMarker) {}
214215

216+
@Override
217+
public void emitRecordAttributes(RecordAttributes recordAttributes) {}
218+
215219
@Override
216220
public void close() {}
217221
};

‎amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/src/test/java/org/apache/amoro/flink/catalog/TestMixedCatalog.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,11 @@ public void before() throws Exception {
116116
@After
117117
public void after() {
118118
sql("DROP TABLE IF EXISTS " + catalogName + "." + DB + "." + TABLE);
119+
// Switch away from the catalog before dropping the database.
120+
// Flink 1.19+ rejects dropping the currently active database.
121+
sql("USE CATALOG default_catalog");
119122
sql("DROP DATABASE IF EXISTS " + catalogName + "." + DB);
120123
Assert.assertTrue(CollectionUtil.isNullOrEmpty(getMixedFormatCatalog().listDatabases()));
121-
sql("USE CATALOG default_catalog");
122124
sql("DROP CATALOG " + catalogName);
123125
}
124126

‎amoro-format-mixed/amoro-mixed-flink/pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
<module>v1.17/amoro-mixed-flink-runtime-1.17</module>
4141
<module>v1.18/amoro-mixed-flink-1.18</module>
4242
<module>v1.18/amoro-mixed-flink-runtime-1.18</module>
43+
<module>v1.19/amoro-mixed-flink-1.19</module>
44+
<module>v1.19/amoro-mixed-flink-runtime-1.19</module>
4345
</modules>
4446

4547
<properties>

‎amoro-format-mixed/amoro-mixed-flink/v1.19/amoro-mixed-flink-1.19/pom.xml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>org.apache.amoro</groupId>
24+
<artifactId>amoro-mixed-flink</artifactId>
25+
<version>0.9-SNAPSHOT</version>
26+
<relativePath>../../pom.xml</relativePath>
27+
</parent>
28+
29+
<artifactId>amoro-format-mixed-flink-1.19</artifactId>
30+
31+
<packaging>jar</packaging>
32+
<name>Amoro Project Mixed Format Flink 1.19</name>
33+
<url>https://amoro.apache.org</url>
34+
35+
<properties>
36+
<kafka.version>3.2.3</kafka.version>
37+
<assertj.version>3.21.0</assertj.version>
38+
<flink.version>1.19.1</flink.version>
39+
</properties>
40+
41+
<dependencies>
42+
<dependency>
43+
<groupId>org.apache.amoro</groupId>
44+
<artifactId>amoro-mixed-flink-common</artifactId>
45+
<version>${project.parent.version}</version>
46+
</dependency>
47+
48+
<dependency>
49+
<groupId>org.apache.iceberg</groupId>
50+
<artifactId>iceberg-flink-1.19</artifactId>
51+
<version>${iceberg.version}</version>
52+
<exclusions>
53+
<exclusion>
54+
<groupId>org.slf4j</groupId>
55+
<artifactId>slf4j-api</artifactId>
56+
</exclusion>
57+
<exclusion>
58+
<groupId>org.apache.parquet</groupId>
59+
<artifactId>parquet-column</artifactId>
60+
</exclusion>
61+
<exclusion>
62+
<groupId>org.apache.parquet</groupId>
63+
<artifactId>parquet-avro</artifactId>
64+
</exclusion>
65+
</exclusions>
66+
</dependency>
67+
68+
<dependency>
69+
<groupId>org.apache.paimon</groupId>
70+
<artifactId>paimon-flink-1.19</artifactId>
71+
<version>${paimon.version}</version>
72+
</dependency>
73+
</dependencies>
74+
75+
<build>
76+
<plugins>
77+
<plugin>
78+
<groupId>org.apache.maven.plugins</groupId>
79+
<artifactId>maven-shade-plugin</artifactId>
80+
<executions>
81+
<execution>
82+
<id>shade-amoro</id>
83+
<goals>
84+
<goal>shade</goal>
85+
</goals>
86+
<phase>package</phase>
87+
<configuration>
88+
<artifactSet>
89+
<includes combine.children="append">
90+
<include>org.apache.amoro:amoro-format-mixed-flink-common</include>
91+
</includes>
92+
</artifactSet>
93+
</configuration>
94+
</execution>
95+
</executions>
96+
</plugin>
97+
</plugins>
98+
</build>
99+
</project>

0 commit comments

Comments
 (0)