Skip to content

Commit 1a6ae6e

Browse files
gfmiogfmio
committed
support flink 1.17.1
1 parent 46cbfd5 commit 1a6ae6e

File tree

12 files changed

+122
-35
lines changed

12 files changed

+122
-35
lines changed

‎pom.xml

Lines changed: 88 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,28 @@ under the License.
7676
<java.version>1.8</java.version>
7777
<spotless-maven-plugin.version>1.20.0</spotless-maven-plugin.version>
7878
<auto-service.version>1.0-rc6</auto-service.version>
79-
<protobuf.version>3.7.1</protobuf.version>
80-
<unixsocket.version>2.3.2</unixsocket.version>
81-
<protoc-jar-maven-plugin.version>3.11.1</protoc-jar-maven-plugin.version>
82-
<flink.version>1.15.2</flink.version>
79+
<protobuf.version>3.23.2</protobuf.version>
80+
<unixsocket.version>2.6.2</unixsocket.version>
81+
<protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
82+
<flink.version>1.17.1</flink.version>
8383
<scala.binary.version>2.12</scala.binary.version>
8484
<scala.version>2.12.7</scala.version>
8585
<lz4-java.version>1.8.0</lz4-java.version>
86-
<flink-shaded-jackson.version>2.12.4-15.0</flink-shaded-jackson.version>
86+
<flink-shaded-jackson.version>2.14.2-17.0</flink-shaded-jackson.version>
8787
<slf4j-log4j12.version>1.7.32</slf4j-log4j12.version>
88+
<flink-connector-kinesis.version>4.1.0-1.17</flink-connector-kinesis.version>
89+
<flink-connector-aws-kinesis-streams.version>4.1.0-1.17</flink-connector-aws-kinesis-streams.version>
90+
<okhttp.version>3.14.6</okhttp.version>
91+
<flink-shaded-netty.version>4.1.82.Final-16.1</flink-shaded-netty.version>
92+
<junit.version>4.12</junit.version>
93+
<hamcrest-all.version>1.3</hamcrest-all.version>
94+
<kryo.version>2.24.0</kryo.version>
95+
<jackson-databind.version>2.13.2.2</jackson-databind.version>
96+
<flink-shaded-netty.version>4.1.82.Final-16.1</flink-shaded-netty.version>
97+
<flink-shaded-force-shading.version>16.1</flink-shaded-force-shading.version>
98+
<commons-codec.version>1.15</commons-codec.version>
99+
<commons-logging.version>1.2</commons-logging.version>
100+
<slf4j-api.version>1.7.36</slf4j-api.version>
88101
<test.unit.pattern>**/*Test.*</test.unit.pattern>
89102
</properties>
90103

@@ -101,15 +114,20 @@ under the License.
101114
<dependency>
102115
<groupId>junit</groupId>
103116
<artifactId>junit</artifactId>
104-
<version>4.12</version>
117+
<version>${junit.version}</version>
105118
<scope>test</scope>
106119
</dependency>
107120
<dependency>
108121
<groupId>org.hamcrest</groupId>
109122
<artifactId>hamcrest-all</artifactId>
110-
<version>1.3</version>
123+
<version>${hamcrest-all.version}</version>
111124
<scope>test</scope>
112125
</dependency>
126+
<dependency>
127+
<groupId>com.google.protobuf</groupId>
128+
<artifactId>protobuf-java</artifactId>
129+
<version>${protobuf.version}</version>
130+
</dependency>
113131
<!--
114132
Resolve dependency convergence issue:
115133
flink-core:1.15.2 depends on kryo:2.24.0
@@ -118,7 +136,7 @@ under the License.
118136
<dependency>
119137
<groupId>com.esotericsoftware.kryo</groupId>
120138
<artifactId>kryo</artifactId>
121-
<version>2.24.0</version>
139+
<version>${kryo.version}</version>
122140
</dependency>
123141
<!--
124142
Resolve dependency convergence issue:
@@ -129,7 +147,66 @@ under the License.
129147
<dependency>
130148
<groupId>com.fasterxml.jackson.core</groupId>
131149
<artifactId>jackson-databind</artifactId>
132-
<version>2.13.2.2</version>
150+
<version>${jackson-databind.version}</version>
151+
</dependency>
152+
<!--
153+
Resolve dependency convergence issue:
154+
org.apache.flink:flink-streaming-java:1.17.1 depends on org.apache.flink:flink-shaded-netty:4.1.82.Final-16.1
155+
org.apache.flink:statefun-flink-core:3.4-SNAPSHOT depends on org.apache.flink:flink-shaded-netty:4.1.70.Final-15.0
156+
(via com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.13.2)
157+
-->
158+
<dependency>
159+
<groupId>org.apache.flink</groupId>
160+
<artifactId>flink-shaded-netty</artifactId>
161+
<version>${flink-shaded-netty.version}</version>
162+
</dependency>
163+
164+
<dependency>
165+
<groupId>org.apache.flink</groupId>
166+
<artifactId>flink-core</artifactId>
167+
<version>${flink.version}</version>
168+
</dependency>
169+
170+
<dependency>
171+
<groupId>org.apache.flink</groupId>
172+
<artifactId>flink-table-common</artifactId>
173+
<version>${flink.version}</version>
174+
</dependency>
175+
176+
<dependency>
177+
<groupId>org.apache.flink</groupId>
178+
<artifactId>flink-connector-base</artifactId>
179+
<version>${flink.version}</version>
180+
</dependency>
181+
182+
<dependency>
183+
<groupId>org.apache.flink</groupId>
184+
<artifactId>flink-shaded-force-shading</artifactId>
185+
<version>${flink-shaded-force-shading.version}</version>
186+
</dependency>
187+
188+
<dependency>
189+
<groupId>org.apache.flink</groupId>
190+
<artifactId>flink-shaded-jackson</artifactId>
191+
<version>${flink-shaded-jackson.version}</version>
192+
</dependency>
193+
194+
<dependency>
195+
<groupId>commons-codec</groupId>
196+
<artifactId>commons-codec</artifactId>
197+
<version>${commons-codec.version}</version>
198+
</dependency>
199+
200+
<dependency>
201+
<groupId>commons-logging</groupId>
202+
<artifactId>commons-logging</artifactId>
203+
<version>${commons-logging.version}</version>
204+
</dependency>
205+
206+
<dependency>
207+
<groupId>org.slf4j</groupId>
208+
<artifactId>slf4j-api</artifactId>
209+
<version>${slf4j-api.version}</version>
133210
</dependency>
134211
</dependencies>
135212
</dependencyManagement>
@@ -234,10 +311,12 @@ under the License.
234311
<outputTarget>
235312
<type>descriptor</type>
236313
<outputDirectory>${basedir}/target/test-classes</outputDirectory>
314+
<addSources>main</addSources>
237315
</outputTarget>
238316
<outputTarget>
239317
<type>java</type>
240318
<outputDirectory>${basedir}/target/generated-test-sources/protoc-jar</outputDirectory>
319+
<addSources>test</addSources>
241320
</outputTarget>
242321
</outputTargets>
243322
</configuration>
@@ -412,4 +491,3 @@ under the License.
412491
</plugins>
413492
</build>
414493
</project>
415-

‎statefun-flink/statefun-flink-core/pom.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@ under the License.
3030
<artifactId>statefun-flink-core</artifactId>
3131

3232
<properties>
33-
<okhttp.version>3.14.6</okhttp.version>
3433
<additional-sources.dir>target/additional-sources</additional-sources.dir>
35-
<flink-shaded-netty.version>4.1.70.Final-15.0</flink-shaded-netty.version>
3634
</properties>
3735

3836
<dependencies>
@@ -90,6 +88,7 @@ under the License.
9088
<groupId>com.kohlschutter.junixsocket</groupId>
9189
<artifactId>junixsocket-core</artifactId>
9290
<version>${unixsocket.version}</version>
91+
<type>pom</type>
9392
</dependency>
9493

9594
<dependency>

‎statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsConfig.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ public void setEmbedded(boolean embedded) {
265265
*/
266266
public StatefulFunctionsUniverseProvider getProvider(ClassLoader cl) {
267267
try {
268-
return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl, false);
268+
return InstantiationUtil.deserializeObject(universeInitializerClassBytes, cl);
269269
} catch (IOException | ClassNotFoundException e) {
270270
throw new IllegalStateException("Unable to initialize.", e);
271271
}

‎statefun-flink/statefun-flink-core/src/main/java/org/apache/flink/statefun/flink/core/StatefulFunctionsJob.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@
2323
import java.util.concurrent.atomic.AtomicInteger;
2424
import org.apache.flink.api.java.utils.ParameterTool;
2525
import org.apache.flink.configuration.Configuration;
26-
import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders;
2726
import org.apache.flink.statefun.flink.core.feedback.FeedbackKey;
2827
import org.apache.flink.statefun.flink.core.message.Message;
2928
import org.apache.flink.statefun.flink.core.translation.FlinkUniverse;
3029
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
3130
import org.apache.flink.util.FlinkUserCodeClassLoader;
31+
import org.apache.flink.util.FlinkUserCodeClassLoaders;
3232

3333
public class StatefulFunctionsJob {
3434

‎statefun-flink/statefun-flink-core/src/test/java/org/apache/flink/statefun/flink/core/functions/ReductionsTest.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,10 +352,11 @@ public boolean deregisterKeySelectionListener(KeySelectionListener<Object> liste
352352

353353
@Nonnull
354354
@Override
355-
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
355+
public <N, SV, SEV, S extends State, IS extends S> IS createOrUpdateInternalState(
356356
@Nonnull TypeSerializer<N> namespaceSerializer,
357357
@Nonnull StateDescriptor<S, SV> stateDesc,
358-
@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
358+
@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory)
359+
throws Exception {
359360
throw new UnsupportedOperationException();
360361
}
361362

‎statefun-flink/statefun-flink-datastream/pom.xml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,13 @@ under the License.
5151
<artifactId>statefun-flink-io</artifactId>
5252
<version>${project.version}</version>
5353
</dependency>
54-
55-
<!-- The following dependencies are here with scope provided, because:
54+
55+
<!-- The following dependencies are here with scope provided, because:
5656
a) they are transitively required by the statefun-flink-* dependencies
57-
b) they are provided at runtime, by the embedding application.
58-
59-
Also note that org.slf4j:slf4j-api is excluded from all the artifacts, since maven
60-
convergence plugging fails.
57+
b) they are provided at runtime, by the embedding application.
58+
59+
Also note that org.slf4j:slf4j-api is excluded from all the artifacts, since maven
60+
convergence plugging fails.
6161
-->
6262
<dependency>
6363
<groupId>org.apache.flink</groupId>
@@ -87,7 +87,7 @@ under the License.
8787
<groupId>org.hamcrest</groupId>
8888
<artifactId>hamcrest-all</artifactId>
8989
</dependency>
90-
90+
9191
</dependencies>
9292

9393
<build>
@@ -152,4 +152,4 @@ under the License.
152152
</plugin>
153153
</plugins>
154154
</build>
155-
</project>
155+
</project>

‎statefun-flink/statefun-flink-distribution/pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ under the License.
7474
<artifactId>statefun-flink-launcher</artifactId>
7575
<version>${project.version}</version>
7676
</dependency>
77-
77+
7878
<!-- flink runtime metrics -->
7979
<dependency>
8080
<groupId>org.apache.flink</groupId>
@@ -199,4 +199,4 @@ under the License.
199199
</plugin>
200200
</plugins>
201201
</build>
202-
</project>
202+
</project>

‎statefun-flink/statefun-flink-io-bundle/pom.xml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,12 @@ under the License.
9090
<dependency>
9191
<groupId>org.apache.flink</groupId>
9292
<artifactId>flink-connector-kinesis</artifactId>
93-
<version>${flink.version}</version>
93+
<version>${flink-connector-kinesis.version}</version>
94+
</dependency>
95+
<dependency>
96+
<groupId>org.apache.flink</groupId>
97+
<artifactId>flink-connector-aws-kinesis-streams</artifactId>
98+
<version>${flink-connector-aws-kinesis-streams.version}</version>
9499
</dependency>
95100

96101
<!-- 3rd party -->
@@ -110,4 +115,4 @@ under the License.
110115
</dependency>
111116
</dependencies>
112117

113-
</project>
118+
</project>

‎statefun-flink/statefun-flink-io/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,4 +124,4 @@ under the License.
124124
</build>
125125

126126

127-
</project>
127+
</project>

‎statefun-shaded/statefun-protobuf-shaded/pom.xml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ under the License.
3232

3333
<properties>
3434
<protobuf-sources.dir>${generated-sources.basedir}/shaded-protobuf-java/</protobuf-sources.dir>
35-
<protobuf.version>3.7.1</protobuf.version>
3635
</properties>
3736

3837
<dependencies>
@@ -107,4 +106,4 @@ under the License.
107106
</plugin>
108107
</plugins>
109108
</build>
110-
</project>
109+
</project>

‎statefun-shaded/statefun-protocol-shaded/pom.xml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ under the License.
3333
<properties>
3434
<protocol-messages.dir>${generated-sources.basedir}/protocol-messages/</protocol-messages.dir>
3535
<proto-sources.dir>target/proto-sources</proto-sources.dir>
36-
<protoc-jar-maven-plugin.version>3.11.1</protoc-jar-maven-plugin.version>
36+
<protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
3737
</properties>
3838

3939
<dependencies>
@@ -47,6 +47,11 @@ under the License.
4747
<artifactId>statefun-protobuf-shaded</artifactId>
4848
<version>${project.version}</version>
4949
</dependency>
50+
<dependency>
51+
<groupId>com.google.protobuf</groupId>
52+
<artifactId>protobuf-java</artifactId>
53+
<version>${protobuf.version}</version>
54+
</dependency>
5055
</dependencies>
5156

5257
<build>
@@ -130,4 +135,4 @@ under the License.
130135
</plugin>
131136
</plugins>
132137
</build>
133-
</project>
138+
</project>

‎tools/docker/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
FROM apache/flink:1.15.2-scala_2.12-java8
16+
FROM flink:1.17.1-scala_2.12-java11
1717

1818
ENV ROLE worker
1919
ENV MASTER_HOST localhost
@@ -45,7 +45,7 @@ RUN mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop && \
4545
# add tcnative
4646
RUN mv $FLINK_HOME/opt/flink-shaded-netty-tcnative-dynamic-*.jar $FLINK_HOME/lib/
4747

48-
# entry point
48+
# entry point
4949
ADD docker-entry-point.sh /docker-entry-point.sh
5050

5151
ENTRYPOINT ["/docker-entry-point.sh"]

0 commit comments

Comments
 (0)