diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml index 204878c4e14..dcb461ee239 100644 --- a/.github/workflows/flink_cdc_base.yml +++ b/.github/workflows/flink_cdc_base.yml @@ -108,6 +108,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 strategy: + fail-fast: false matrix: java-version: ${{ fromJSON(inputs.java-version) }} flink-version: ${{ fromJSON(inputs.flink-version) }} @@ -148,7 +149,7 @@ jobs: maven-version: 3.8.6 - name: Compile and test - timeout-minutes: 90 + timeout-minutes: 60 run: | set -o pipefail diff --git a/.github/workflows/flink_cdc_ci.yml b/.github/workflows/flink_cdc_ci.yml index 76e7691e3f8..47f1c77c27f 100644 --- a/.github/workflows/flink_cdc_ci.yml +++ b/.github/workflows/flink_cdc_ci.yml @@ -59,6 +59,7 @@ jobs: run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb ut: strategy: + fail-fast: false matrix: module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ] name: Unit Tests @@ -68,6 +69,7 @@ jobs: module: ${{ matrix.module }} pipeline_e2e: strategy: + fail-fast: false matrix: parallelism: [ 1, 4 ] name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism) diff --git a/.github/workflows/flink_cdc_ci_nightly.yml b/.github/workflows/flink_cdc_ci_nightly.yml index 456c5f8926c..c69238f422e 100644 --- a/.github/workflows/flink_cdc_ci_nightly.yml +++ b/.github/workflows/flink_cdc_ci_nightly.yml @@ -48,6 +48,7 @@ jobs: run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb ut: strategy: + fail-fast: false matrix: module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ] name: Unit Tests @@ -57,6 +58,7 @@ jobs: module: ${{ matrix.module }} pipeline_e2e: strategy: + fail-fast: false matrix: parallelism: [ 1, 4 ] name: Pipeline E2E Tests (${{ matrix.parallelism }} Parallelism) diff --git a/.github/workflows/flink_cdc_migration_test_base.yml b/.github/workflows/flink_cdc_migration_test_base.yml index fa8012fd3fa..ff707c0fd8e 100644 --- a/.github/workflows/flink_cdc_migration_test_base.yml +++ b/.github/workflows/flink_cdc_migration_test_base.yml @@ -32,6 +32,7 @@ jobs: migration_test_ut: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: java-version: ${{ fromJSON(inputs.java-version) }} flink-version: ${{ fromJSON(inputs.flink-version) }} @@ -52,6 +53,7 @@ jobs: pipeline_migration_test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: java-version: ${{ fromJSON(inputs.java-version) }} flink-version: ${{ fromJSON(inputs.flink-version) }} @@ -91,6 +93,7 @@ jobs: data_stream_migration_test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: java-version: ${{ fromJSON(inputs.java-version) }} flink-version: [ '1.19.1', '1.20.0' ] diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml index 8e6886e156a..96366a9af91 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml @@ -98,13 +98,6 @@ limitations under the License. test - - com.alibaba - fastjson - 1.2.83 - test - - diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceITCase.java index de7acd86a71..b562c78293b 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/LegacyMySqlSourceITCase.java @@ -28,8 +28,11 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.kafka.connect.json.JsonConverterConfig; +import org.assertj.core.api.Assertions; import org.junit.Test; import java.io.IOException; @@ -46,8 +49,6 @@ import java.util.Map; import java.util.Objects; -import static org.junit.Assert.assertTrue; - /** Integration tests for the legacy {@link MySqlSource}. */ public class LegacyMySqlSourceITCase extends LegacyMySqlTestBase { @@ -99,9 +100,9 @@ private void testConsumingAllEventsWithJsonFormat( StreamTableEnvironment.create( env, EnvironmentSettings.newInstance().inStreamingMode().build()); - final JSONObject expected = - JSONObject.parseObject(readLines(expectedFile), JSONObject.class); - JSONObject expectSnapshot = expected.getJSONObject("expected_snapshot"); + final JsonNode expected = + new ObjectMapper().readValue(readLines(expectedFile), JsonNode.class); + JsonNode expectSnapshot = expected.get("expected_snapshot"); DataStreamSource source = env.addSource(sourceFunction); tEnv.createTemporaryView("full_types", source); @@ -110,9 +111,8 @@ private void testConsumingAllEventsWithJsonFormat( // check the snapshot result CloseableIterator snapshot = result.collect(); waitForSnapshotStarted(snapshot); - assertTrue( - dataInJsonIsEquals( - fetchRows(snapshot, 1).get(0).toString(), expectSnapshot.toString())); + + assertJsonEquals(extractJsonBody(snapshot.next()), expectSnapshot); try (Connection connection = fullTypesDatabase.getJdbcConnection(); Statement statement = connection.createStatement()) { statement.execute( @@ -121,10 +121,8 @@ private void testConsumingAllEventsWithJsonFormat( // check the binlog result CloseableIterator binlog = result.collect(); - JSONObject expectBinlog = expected.getJSONObject("expected_binlog"); - assertTrue( - dataInJsonIsEquals( - fetchRows(binlog, 1).get(0).toString(), expectBinlog.toString())); + JsonNode expectBinlog = expected.get("expected_binlog"); + assertJsonEquals(extractJsonBody(binlog.next()), expectBinlog); result.getJobClient().get().cancel().get(); } @@ -164,25 +162,23 @@ private static byte[] readLines(String resource) throws IOException, URISyntaxEx return Files.readAllBytes(path); } - private static boolean dataInJsonIsEquals(String actual, String expect) { - JSONObject actualJsonObject = JSONObject.parseObject(actual); - JSONObject expectJsonObject = JSONObject.parseObject(expect); - - if (expectJsonObject.getJSONObject("payload") != null - && actualJsonObject.getJSONObject("payload") != null) { - expectJsonObject = expectJsonObject.getJSONObject("payload"); - actualJsonObject = actualJsonObject.getJSONObject("payload"); + private static void assertJsonEquals(JsonNode actual, JsonNode expect) throws Exception { + if (actual.get("payload") != null && expect.get("payload") != null) { + actual = actual.get("payload"); + expect = expect.get("payload"); } - return jsonObjectEquals( - expectJsonObject.getJSONObject("after"), - actualJsonObject.getJSONObject("after")) - && jsonObjectEquals( - expectJsonObject.getJSONObject("before"), - actualJsonObject.getJSONObject("before")) - && Objects.equals(expectJsonObject.get("op"), actualJsonObject.get("op")); + Assertions.assertThat(actual.get("after")).isEqualTo(expect.get("after")); + Assertions.assertThat(actual.get("before")).isEqualTo(expect.get("before")); + Assertions.assertThat(actual.get("op")).isEqualTo(expect.get("op")); } - private static boolean jsonObjectEquals(JSONObject a, JSONObject b) { - return (a == b) || (a != null && a.toString().equals(b.toString())); + private static JsonNode extractJsonBody(Row row) { + try { + String body = row.toString(); + return new ObjectMapper() + .readValue(body.substring(3, body.length() - 1), JsonNode.class); + } catch (JsonProcessingException e) { + throw new RuntimeException("Invalid JSON format.", e); + } } } diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxCharsetITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxCharsetITCase.java index d4ecb5d974f..d8507c47772 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxCharsetITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxCharsetITCase.java @@ -167,8 +167,8 @@ public void testCharset() throws Exception { + " 'scan.incremental.snapshot.chunk.size' = '%s'" + ")", testName, - HOST_NAME, - PORT, + getHost(), + getPort(), USER_NAME, PASSWORD, DATABASE, diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceITCase.java index db87a98acb4..901a424fbef 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceITCase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceITCase.java @@ -83,8 +83,8 @@ public void testSingleKey() throws Exception { + " 'server-time-zone' = 'UTC'," + " 'server-id' = '%s'" + ")", - HOST_NAME, - PORT, + getHost(), + getPort(), USER_NAME, PASSWORD, DATABASE, @@ -234,8 +234,8 @@ public void testFullTypesDdl() throws Exception { + " 'server-time-zone' = 'UTC'," + " 'server-id' = '%s'" + ")", - HOST_NAME, - PORT, + getHost(), + getPort(), USER_NAME, PASSWORD, DATABASE, @@ -301,8 +301,8 @@ public void testMultiKeys() throws Exception { + " 'server-time-zone' = 'UTC'," + " 'server-id' = '%s'" + ")", - HOST_NAME, - PORT, + getHost(), + getPort(), USER_NAME, PASSWORD, DATABASE, diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceTestBase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceTestBase.java index 43c3c678393..65149c8834d 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceTestBase.java +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceTestBase.java @@ -17,13 +17,11 @@ package org.apache.flink.cdc.connectors.polardbx; +import org.apache.flink.cdc.common.utils.TestCaseUtils; import org.apache.flink.table.planner.factories.TestValuesTableFactory; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.PortBinding; -import com.github.dockerjava.api.model.Ports; import org.apache.commons.lang3.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -64,36 +62,37 @@ public abstract class PolardbxSourceTestBase extends AbstractTestBase { private static final Logger LOG = LoggerFactory.getLogger(PolardbxSourceTestBase.class); private static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$"); - protected static final Integer PORT = 8527; - protected static final String HOST_NAME = "127.0.0.1"; - protected static final String USER_NAME = "polardbx_root"; - protected static final String PASSWORD = "123456"; + private static final String IMAGE_VERSION = "2.1.0"; private static final DockerImageName POLARDBX_IMAGE = DockerImageName.parse("polardbx/polardb-x:" + IMAGE_VERSION); + protected static final Integer INNER_PORT = 8527; + protected static final String USER_NAME = "polardbx_root"; + protected static final String PASSWORD = "123456"; + protected static final Duration WAITING_TIMEOUT = Duration.ofMinutes(1); + protected static final GenericContainer POLARDBX_CONTAINER = new GenericContainer<>(POLARDBX_IMAGE) - .withExposedPorts(PORT) + .withExposedPorts(INNER_PORT) .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withStartupTimeout(Duration.ofMinutes(3)) - .withCreateContainerCmdModifier( - c -> - c.withPortBindings( - new PortBinding( - Ports.Binding.bindPort(PORT), - new ExposedPort(PORT)))); + .withStartupTimeout(Duration.ofMinutes(3)); + + protected static String getHost() { + return POLARDBX_CONTAINER.getHost(); + } + + protected static int getPort() { + return POLARDBX_CONTAINER.getMappedPort(INNER_PORT); + } @BeforeClass - public static void startContainers() throws InterruptedException { - // no need to start container when the port 8527 is listening - if (!checkConnection()) { - LOG.info("Polardbx connection is not valid, so try to start containers..."); - Startables.deepStart(Stream.of(POLARDBX_CONTAINER)).join(); - LOG.info("Containers are started."); - // here should wait 10s that make sure the polardbx is ready - Thread.sleep(10 * 1000); - } + public static void startContainers() { + Startables.deepStart(Stream.of(POLARDBX_CONTAINER)).join(); + LOG.info("Containers are started."); + + TestCaseUtils.repeatedCheck( + PolardbxSourceTestBase::checkConnection, WAITING_TIMEOUT, Duration.ofSeconds(1)); } @AfterClass @@ -104,7 +103,7 @@ public static void stopContainers() { } protected static String getJdbcUrl() { - return String.format("jdbc:mysql://%s:%s", HOST_NAME, PORT); + return String.format("jdbc:mysql://%s:%s", getHost(), getPort()); } protected static Connection getJdbcConnection() throws SQLException { diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json index f11a4cb3962..20c641df522 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude-with-numeric-decimal.json @@ -52,14 +52,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\": \"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "op": "r", "transaction": null @@ -116,14 +116,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "after": { "id": 1, @@ -176,14 +176,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "op": "u", "transaction": null diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json index 3e5dedd08ae..41cc94b3728 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-exclude.json @@ -52,14 +52,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\": \"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "op": "r", "transaction": null @@ -116,14 +116,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "after": { "id": "AQ==", @@ -176,14 +176,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "op": "u", "transaction": null diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json index fed8c09a940..74dc65456d4 100644 --- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json +++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/resources/file/debezium-data-schema-include.json @@ -725,14 +725,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\": \"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "source": { "version": "1.5.2.Final", @@ -1080,14 +1080,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "after": { "id": "AQ==", @@ -1140,14 +1140,14 @@ "enum_c": "red", "set_c": "a,b", "json_c": "{\"key1\":\"value1\"}", - "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/"}, - "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA"}, - "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/"}, - "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA"}, - "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA="}, - "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA"}, - "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA="} + "point_c": {"x":1.0,"y":1.0,"wkb":"AQEAAAAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "geometry_c":{"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "linestring_c": {"wkb":"AQIAAAADAAAAAAAAAAAACEAAAAAAAAAAAAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRA","srid":null}, + "polygon_c": {"wkb":"AQMAAAABAAAABQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAQAAAAAAAAPA/AAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/","srid":null}, + "multipoint_c": {"wkb":"AQQAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQEAAAAAAAAAAAAAQAAAAAAAAABA","srid":null}, + "multiline_c": {"wkb":"AQUAAAACAAAAAQIAAAADAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAAAIQAAAAAAAAAhAAQIAAAACAAAAAAAAAAAAEEAAAAAAAAAQQAAAAAAAABRAAAAAAAAAFEA=","srid":null}, + "multipolygon_c": {"wkb":"AQYAAAACAAAAAQMAAAABAAAABQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAkQAAAAAAAAAAAAAAAAAAAJEAAAAAAAAAAAAAAAAAAAAAAAQMAAAABAAAABQAAAAAAAAAAABRAAAAAAAAAFEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAcQAAAAAAAABRAAAAAAAAAHEAAAAAAAAAUQAAAAAAAABRA","srid":null}, + "geometrycollection_c": {"wkb":"AQcAAAADAAAAAQEAAAAAAAAAAAAkQAAAAAAAACRAAQEAAAAAAAAAAAA+QAAAAAAAAD5AAQIAAAACAAAAAAAAAAAALkAAAAAAAAAuQAAAAAAAADRAAAAAAAAANEA=","srid":null} }, "source": { "version": "1.5.2.Final", diff --git a/tools/mig-test/datastream/compile_jobs.rb b/tools/mig-test/datastream/compile_jobs.rb index 5c906e5bf31..2326a454059 100644 --- a/tools/mig-test/datastream/compile_jobs.rb +++ b/tools/mig-test/datastream/compile_jobs.rb @@ -20,7 +20,7 @@ JOB_VERSIONS.each do |version| puts "Compiling DataStream job for CDC #{version}" - `cd datastream-#{version} && mvn clean package -DskipTests` + system "cd datastream-#{version} && mvn clean package -DskipTests" end puts 'Done' \ No newline at end of file diff --git a/tools/mig-test/datastream/datastream-3.2.0/pom.xml b/tools/mig-test/datastream/datastream-3.2.0/pom.xml index c1f556033d1..b624439edb3 100644 --- a/tools/mig-test/datastream/datastream-3.2.0/pom.xml +++ b/tools/mig-test/datastream/datastream-3.2.0/pom.xml @@ -29,7 +29,7 @@ limitations under the License. UTF-8 1.18.1 3.2.0 - 1.9.7.Final + 1.9.8.Final 2.12 2.0.13 UTF-8 @@ -136,20 +136,95 @@ limitations under the License. - maven-assembly-plugin + org.apache.maven.plugins + maven-shade-plugin + shade-flink package - single + shade + + + false + false + true + + ${project.basedir}/target/dependency-reduced-pom.xml + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + io.debezium:debezium-api + io.debezium:debezium-embedded + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-connector-mysql + org.apache.flink:flink-connector-debezium + org.apache.flink:flink-connector-mysql-cdc + org.antlr:antlr4-runtime + org.apache.kafka:* + mysql:mysql-connector-java + com.zendesk:mysql-binlog-connector-java + com.fasterxml.*:* + com.google.guava:* + com.esri.geometry:esri-geometry-api + com.zaxxer:HikariCP + + org.apache.flink:flink-shaded-guava + + + + + org.apache.kafka + + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + + + + org.antlr + + org.apache.flink.cdc.connectors.shaded.org.antlr + + + + com.fasterxml + + org.apache.flink.cdc.connectors.shaded.com.fasterxml + + + + com.google + + org.apache.flink.cdc.connectors.shaded.com.google + + + + com.esri.geometry + org.apache.flink.cdc.connectors.shaded.com.esri.geometry + + + com.zaxxer + + org.apache.flink.cdc.connectors.shaded.com.zaxxer + + + + - - - jar-with-dependencies - - diff --git a/tools/mig-test/datastream/datastream-3.2.0/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.2.0/src/main/java/DataStreamJob.java index f821ac0a2de..bfaa2d529bf 100644 --- a/tools/mig-test/datastream/datastream-3.2.0/src/main/java/DataStreamJob.java +++ b/tools/mig-test/datastream/datastream-3.2.0/src/main/java/DataStreamJob.java @@ -23,7 +23,7 @@ public class DataStreamJob { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MySqlSource mySqlSource = MySqlSource.builder() .hostname("localhost") .port(3306) @@ -45,10 +45,6 @@ public static void main(String[] args) { .print() .setParallelism(1); - try { - env.execute(); - } catch (Exception e) { - // ... unfortunately - } + env.execute(); } } diff --git a/tools/mig-test/datastream/datastream-3.2.1/pom.xml b/tools/mig-test/datastream/datastream-3.2.1/pom.xml index c7d680a2f3b..08c7337af04 100644 --- a/tools/mig-test/datastream/datastream-3.2.1/pom.xml +++ b/tools/mig-test/datastream/datastream-3.2.1/pom.xml @@ -136,20 +136,95 @@ limitations under the License. - maven-assembly-plugin + org.apache.maven.plugins + maven-shade-plugin + shade-flink package - single + shade + + + false + false + true + + ${project.basedir}/target/dependency-reduced-pom.xml + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + io.debezium:debezium-api + io.debezium:debezium-embedded + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-connector-mysql + org.apache.flink:flink-connector-debezium + org.apache.flink:flink-connector-mysql-cdc + org.antlr:antlr4-runtime + org.apache.kafka:* + mysql:mysql-connector-java + com.zendesk:mysql-binlog-connector-java + com.fasterxml.*:* + com.google.guava:* + com.esri.geometry:esri-geometry-api + com.zaxxer:HikariCP + + org.apache.flink:flink-shaded-guava + + + + + org.apache.kafka + + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + + + + org.antlr + + org.apache.flink.cdc.connectors.shaded.org.antlr + + + + com.fasterxml + + org.apache.flink.cdc.connectors.shaded.com.fasterxml + + + + com.google + + org.apache.flink.cdc.connectors.shaded.com.google + + + + com.esri.geometry + org.apache.flink.cdc.connectors.shaded.com.esri.geometry + + + com.zaxxer + + org.apache.flink.cdc.connectors.shaded.com.zaxxer + + + + - - - jar-with-dependencies - - diff --git a/tools/mig-test/datastream/datastream-3.2.1/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.2.1/src/main/java/DataStreamJob.java index f821ac0a2de..bfaa2d529bf 100644 --- a/tools/mig-test/datastream/datastream-3.2.1/src/main/java/DataStreamJob.java +++ b/tools/mig-test/datastream/datastream-3.2.1/src/main/java/DataStreamJob.java @@ -23,7 +23,7 @@ public class DataStreamJob { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MySqlSource mySqlSource = MySqlSource.builder() .hostname("localhost") .port(3306) @@ -45,10 +45,6 @@ public static void main(String[] args) { .print() .setParallelism(1); - try { - env.execute(); - } catch (Exception e) { - // ... unfortunately - } + env.execute(); } } diff --git a/tools/mig-test/datastream/datastream-3.3.0/pom.xml b/tools/mig-test/datastream/datastream-3.3.0/pom.xml index cc65c6c7869..63bcd21b98d 100644 --- a/tools/mig-test/datastream/datastream-3.3.0/pom.xml +++ b/tools/mig-test/datastream/datastream-3.3.0/pom.xml @@ -29,7 +29,7 @@ limitations under the License. UTF-8 1.19.1 3.3.0 - 1.9.7.Final + 1.9.8.Final 2.12 2.0.13 UTF-8 @@ -136,20 +136,95 @@ limitations under the License. - maven-assembly-plugin + org.apache.maven.plugins + maven-shade-plugin + shade-flink package - single + shade + + + false + false + true + + ${project.basedir}/target/dependency-reduced-pom.xml + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + io.debezium:debezium-api + io.debezium:debezium-embedded + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-connector-mysql + org.apache.flink:flink-connector-debezium + org.apache.flink:flink-connector-mysql-cdc + org.antlr:antlr4-runtime + org.apache.kafka:* + mysql:mysql-connector-java + com.zendesk:mysql-binlog-connector-java + com.fasterxml.*:* + com.google.guava:* + com.esri.geometry:esri-geometry-api + com.zaxxer:HikariCP + + org.apache.flink:flink-shaded-guava + + + + + org.apache.kafka + + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + + + + org.antlr + + org.apache.flink.cdc.connectors.shaded.org.antlr + + + + com.fasterxml + + org.apache.flink.cdc.connectors.shaded.com.fasterxml + + + + com.google + + org.apache.flink.cdc.connectors.shaded.com.google + + + + com.esri.geometry + org.apache.flink.cdc.connectors.shaded.com.esri.geometry + + + com.zaxxer + + org.apache.flink.cdc.connectors.shaded.com.zaxxer + + + + - - - jar-with-dependencies - - diff --git a/tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java index f821ac0a2de..bfaa2d529bf 100644 --- a/tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java +++ b/tools/mig-test/datastream/datastream-3.3.0/src/main/java/DataStreamJob.java @@ -23,7 +23,7 @@ public class DataStreamJob { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MySqlSource mySqlSource = MySqlSource.builder() .hostname("localhost") .port(3306) @@ -45,10 +45,6 @@ public static void main(String[] args) { .print() .setParallelism(1); - try { - env.execute(); - } catch (Exception e) { - // ... unfortunately - } + env.execute(); } } diff --git a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml index e174d5583cf..6d73f0e0430 100644 --- a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml +++ b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/pom.xml @@ -29,7 +29,7 @@ limitations under the License. UTF-8 1.19.1 3.4-SNAPSHOT - 1.9.7.Final + 1.9.8.Final 2.12 2.0.13 UTF-8 @@ -136,20 +136,95 @@ limitations under the License. - maven-assembly-plugin + org.apache.maven.plugins + maven-shade-plugin + shade-flink package - single + shade + + + false + false + true + + ${project.basedir}/target/dependency-reduced-pom.xml + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + io.debezium:debezium-api + io.debezium:debezium-embedded + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-connector-mysql + org.apache.flink:flink-connector-debezium + org.apache.flink:flink-connector-mysql-cdc + org.antlr:antlr4-runtime + org.apache.kafka:* + mysql:mysql-connector-java + com.zendesk:mysql-binlog-connector-java + com.fasterxml.*:* + com.google.guava:* + com.esri.geometry:esri-geometry-api + com.zaxxer:HikariCP + + org.apache.flink:flink-shaded-guava + + + + + org.apache.kafka + + org.apache.flink.cdc.connectors.shaded.org.apache.kafka + + + + org.antlr + + org.apache.flink.cdc.connectors.shaded.org.antlr + + + + com.fasterxml + + org.apache.flink.cdc.connectors.shaded.com.fasterxml + + + + com.google + + org.apache.flink.cdc.connectors.shaded.com.google + + + + com.esri.geometry + org.apache.flink.cdc.connectors.shaded.com.esri.geometry + + + com.zaxxer + + org.apache.flink.cdc.connectors.shaded.com.zaxxer + + + + - - - jar-with-dependencies - - diff --git a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java index f821ac0a2de..bfaa2d529bf 100644 --- a/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java +++ b/tools/mig-test/datastream/datastream-3.4-SNAPSHOT/src/main/java/DataStreamJob.java @@ -23,7 +23,7 @@ public class DataStreamJob { - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MySqlSource mySqlSource = MySqlSource.builder() .hostname("localhost") .port(3306) @@ -45,10 +45,6 @@ public static void main(String[] args) { .print() .setParallelism(1); - try { - env.execute(); - } catch (Exception e) { - // ... unfortunately - } + env.execute(); } } diff --git a/tools/mig-test/datastream/run_migration_test.rb b/tools/mig-test/datastream/run_migration_test.rb index deb16b0f074..951d043daad 100644 --- a/tools/mig-test/datastream/run_migration_test.rb +++ b/tools/mig-test/datastream/run_migration_test.rb @@ -32,12 +32,18 @@ def exec_sql_source(sql) `mysql -h 127.0.0.1 -P#{SOURCE_PORT} -uroot --skip-password -e "USE #{DATABASE_NAME}; #{sql}"` end +def extract_job_id(output) + current_job_id = output.split("\n").filter { _1.start_with?('Job has been submitted with JobID ') }.first&.split&.last + raise StandardError, "Failed to submit Flink job. Output: #{output}" unless current_job_id&.length == 32 + current_job_id +end + def put_mystery_data(mystery) exec_sql_source("REPLACE INTO girl(id, name) VALUES (17, '#{mystery}');") end def ensure_mystery_data(mystery) - throw StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery + raise StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery end puts ' Waiting for source to start up...' @@ -52,8 +58,8 @@ def test_migration_chore(from_version, to_version) # Clear previous savepoints and logs `rm -rf savepoints` - old_job_id = `#{FLINK_HOME}/bin/flink run -p 1 -c DataStreamJob --detached datastream-#{from_version}/target/datastream-job-#{from_version}-jar-with-dependencies.jar`.split.last - raise StandardError, 'Failed to submit Flink job' unless old_job_id.length == 32 + old_output = `#{FLINK_HOME}/bin/flink run -p 1 -c DataStreamJob --detached datastream-#{from_version}/target/datastream-job-#{from_version}.jar` + old_job_id = extract_job_id(old_output) puts "Submitted job at #{from_version} as #{old_job_id}" @@ -64,8 +70,8 @@ def test_migration_chore(from_version, to_version) puts `#{FLINK_HOME}/bin/flink stop --savepointPath #{Dir.pwd}/savepoints #{old_job_id}` savepoint_file = `ls savepoints`.split("\n").last - new_job_id = `#{FLINK_HOME}/bin/flink run --fromSavepoint #{Dir.pwd}/savepoints/#{savepoint_file} -p 1 -c DataStreamJob --detached datastream-#{to_version}/target/datastream-job-#{to_version}-jar-with-dependencies.jar`.split.last - raise StandardError, 'Failed to submit Flink job' unless new_job_id.length == 32 + new_output = `#{FLINK_HOME}/bin/flink run --fromSavepoint #{Dir.pwd}/savepoints/#{savepoint_file} -p 1 -c DataStreamJob --detached datastream-#{to_version}/target/datastream-job-#{to_version}.jar` + new_job_id = extract_job_id(new_output) puts "Submitted job at #{to_version} as #{new_job_id}" random_string_2 = SecureRandom.hex(8) @@ -94,45 +100,25 @@ def test_migration(from_version, to_version) end version_list = %w[3.2.0 3.2.1 3.3.0 3.4-SNAPSHOT] -version_result = Hash.new('❓') @failures = [] -version_list.each_with_index do |old_version, old_index| +new_version = version_list.last + +version_list.each do |old_version| + puts "-> Testing migrating from #{old_version} to latest snapshot." puts 'Restarting cluster...' `#{FLINK_HOME}/bin/stop-cluster.sh` `rm -rf #{FLINK_HOME}/log/flink-*.out` + puts 'Stopped cluster.' `#{FLINK_HOME}/bin/start-cluster.sh` - version_list.each_with_index do |new_version, new_index| - next if old_index > new_index - - result = test_migration old_version, new_version - version_result[old_version + new_version] = result ? '✅' : '❌' - @failures << [old_version, new_version] unless result - end -end - -printable_result = [] -printable_result << [''] + version_list -version_list.each_with_index do |old_version, old_index| - table_line = [old_version] - version_list.each_with_index do |new_version, new_index| - table_line << if old_index > new_index - '' - else - version_result[old_version + new_version] - end - end - printable_result << table_line -end + puts 'Started cluster.' -begin - require 'terminal-table' - puts Terminal::Table.new rows: printable_result, title: 'Migration Test Result' -rescue LoadError - puts 'Test summary: ', printable_result + result = test_migration old_version, new_version + @failures << [old_version, new_version] unless result end -puts "✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`" -if @failures.filter { |_, new_version| new_version == version_list.last }.any? +if @failures.any? + puts 'Some migration to snapshot version tests failed. Details: ' + puts @failures abort 'Some migration to snapshot version tests failed.' end diff --git a/tools/mig-test/run_migration_test.rb b/tools/mig-test/run_migration_test.rb index bb111252cbf..358337762db 100644 --- a/tools/mig-test/run_migration_test.rb +++ b/tools/mig-test/run_migration_test.rb @@ -37,7 +37,7 @@ def put_mystery_data(mystery) end def ensure_mystery_data(mystery) - throw StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery + raise StandardError, 'Failed to get specific mystery string' unless `cat #{FLINK_HOME}/log/*.out`.include? mystery end def extract_job_id(output) @@ -120,15 +120,14 @@ def test_migration(from_version, to_version) version_list = case ARGV[0] when '1.19.1' then %w[3.2.0 3.2.1 3.3.0 3.4-SNAPSHOT] when '1.20.0' then %w[3.2.1 3.3.0 3.4-SNAPSHOT] - else [] + else throw "Unknown Flink target version: #{ARGV[0]}." end -version_result = Hash.new('❓') @failures = [] new_version = version_list.last -version_list.each_with_index do |old_version, old_index| +version_list.each do |old_version| puts "-> Testing migrating from #{old_version} to latest snapshot." puts 'Restarting cluster...' `#{FLINK_HOME}/bin/stop-cluster.sh` @@ -137,7 +136,6 @@ def test_migration(from_version, to_version) puts 'Started cluster.' result = test_migration old_version, new_version - version_result[old_version + new_version] = result ? '✅' : '❌' @failures << "#{old_version} => #{new_version}" unless result end