From 0118330103e992253a7875a523de3526cad4a6c5 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 6 Sep 2021 08:55:52 -0700 Subject: [PATCH] KAFKA-13273: Add support for Java 17 (#11296) Java 17 is at release candidate stage and it will be a LTS release once it's out (previous LTS release was Java 11). Details: * Replace Java 16 with Java 17 in Jenkins and Readme. * Replace `--illegal-access=permit` (which was removed from Java 17) with `--add-opens` for the packages we require internal access to. Filed KAFKA-13275 for updating the tests not to require `--add-opens` (where possible). * Update `release.py` to use JDK8. and JDK 17 (instead of JDK 8 and JDK 15). * Removed all but one Streams test from `testsToExclude`. The Connect test exclusion list remains the same. * Add notable change to upgrade.html * Upgrade to Gradle 7.2 as it's required for proper Java 17 support. * Upgrade mockito to 3.12.4 for better Java 17 support. * Adjusted `KafkaRaftClientTest` and `QuorumStateTest` not to require private access to `jdk.internal.util.random`. Reviewers: Manikumar Reddy , Chia-Ping Tsai --- Jenkinsfile | 12 ++--- README.md | 2 +- build.gradle | 23 ++++++-- docs/upgrade.html | 7 ++- gradle/dependencies.gradle | 4 +- gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 4 +- .../kafka/raft/KafkaRaftClientTest.java | 53 ++++++------------- .../org/apache/kafka/raft/MockableRandom.java | 51 ++++++++++++++++++ .../apache/kafka/raft/QuorumStateTest.java | 26 +++++---- .../kafka/raft/RaftClientTestContext.java | 8 ++- release.py | 10 ++-- 12 files changed, 125 insertions(+), 79 deletions(-) create mode 100644 raft/src/test/java/org/apache/kafka/raft/MockableRandom.java diff --git a/Jenkinsfile b/Jenkinsfile index cfdf72268f363..6b75cfd966110 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,10 +142,10 @@ pipeline { } } - stage('JDK 16 and Scala 2.13') { + stage('JDK 17 and Scala 2.13') { agent { label 'ubuntu' } tools { - jdk 'jdk_16_latest' + jdk 'jdk_17_latest' } options { timeout(time: 8, unit: 'HOURS') @@ -157,7 +157,7 @@ pipeline { steps { doValidation() doTest(env) - echo 'Skipping Kafka Streams archetype test for Java 16' + echo 'Skipping Kafka Streams archetype test for Java 17' } } @@ -231,14 +231,14 @@ pipeline { } } - stage('JDK 16 and Scala 2.12') { + stage('JDK 17 and Scala 2.12') { when { not { changeRequest() } beforeAgent true } agent { label 'ubuntu' } tools { - jdk 'jdk_16_latest' + jdk 'jdk_17_latest' } options { timeout(time: 8, unit: 'HOURS') @@ -250,7 +250,7 @@ pipeline { steps { doValidation() doTest(env) - echo 'Skipping Kafka Streams archetype test for Java 16' + echo 'Skipping Kafka Streams archetype test for Java 17' } } } diff --git a/README.md b/README.md index 41d2b8d8f0013..5c5f33417e38e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ See our [web site](https://kafka.apache.org) for details on the project. You need to have [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) installed. -We build and test Apache Kafka with Java 8, 11 and 16. We set the `release` parameter in javac and scalac +We build and test Apache Kafka with Java 8, 11 and 17. We set the `release` parameter in javac and scalac to `8` to ensure the generated binaries are compatible with Java 8 or higher (independently of the Java version used for compilation). Java 8 support has been deprecated since Apache Kafka 3.0 and will be removed in Apache Kafka 4.0 (see [KIP-750](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=181308223) for more details). diff --git a/build.gradle b/build.gradle index 2b7bc4626f52f..ec6be4c201b3d 100644 --- a/build.gradle +++ b/build.gradle @@ -102,8 +102,22 @@ ext { defaultMaxHeapSize = "2g" defaultJvmArgs = ["-Xss4m", "-XX:+UseParallelGC"] - if (JavaVersion.current() == JavaVersion.VERSION_16) - defaultJvmArgs.add("--illegal-access=permit") + + // "JEP 403: Strongly Encapsulate JDK Internals" causes some tests to fail when they try + // to access internals (often via mocking libraries). We use `--add-opens` as a workaround + // for now and we'll fix it properly (where possible) via KAFKA-13275. + if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16)) + defaultJvmArgs.addAll( + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.nio.file=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.regex=ALL-UNNAMED", + "--add-opens=java.base/java.util.stream=ALL-UNNAMED", + "--add-opens=java.base/java.text=ALL-UNNAMED", + "--add-opens=java.base/java.time=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED" + ) userMaxForks = project.hasProperty('maxParallelForks') ? maxParallelForks.toInteger() : null userIgnoreFailures = project.hasProperty('ignoreFailures') ? ignoreFailures : false @@ -359,7 +373,7 @@ subprojects { // The suites are for running sets of tests in IDEs. // Gradle will run each test class, so we exclude the suites to avoid redundantly running the tests twice. def testsToExclude = ['**/*Suite.class'] - // Exclude PowerMock tests when running with Java 16 until a version of PowerMock that supports Java 16 is released + // Exclude PowerMock tests when running with Java 16 or newer until a version of PowerMock that supports the relevant versions is released // The relevant issues are https://github.com/powermock/powermock/issues/1094 and https://github.com/powermock/powermock/issues/1099 if (JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_16)) { testsToExclude.addAll([ @@ -372,8 +386,7 @@ subprojects { "**/WorkerSinkTaskTest.*", "**/WorkerSinkTaskThreadedTest.*", "**/WorkerSourceTaskTest.*", "**/WorkerTaskTest.*", "**/WorkerTest.*", "**/RestServerTest.*", // streams tests - "**/KafkaStreamsTest.*", "**/RepartitionTopicsTest.*", "**/RocksDBMetricsRecorderTest.*", - "**/StreamsMetricsImplTest.*", "**/StateManagerUtilTest.*", "**/TableSourceNodeTest.*" + "**/KafkaStreamsTest.*" ]) } diff --git a/docs/upgrade.html b/docs/upgrade.html index da1bc496ec3a9..48219d5e03aac 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -19,7 +19,12 @@