Skip to content

Commit

Permalink
add: phaker -> values test
Browse files Browse the repository at this point in the history
  • Loading branch information
yuxiqian committed Aug 14, 2024
1 parent b91b3a9 commit b72249c
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 3 deletions.
4 changes: 3 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,7 @@ libraryDependencies ++= Seq(
"org.apache.flink" % "flink-cdc-runtime" % flinkCdcVersion,
"org.scalatest" %% "scalatest" % "3.2.19" % Test,
"org.apache.flink" % "flink-clients" % flinkVersion % Test,
"org.apache.flink" % "flink-streaming-java" % flinkVersion % Test
"org.apache.flink" % "flink-streaming-java" % flinkVersion % Test,
"org.apache.flink" % "flink-cdc-composer" % flinkCdcVersion % Test,
"org.apache.flink" % "flink-cdc-pipeline-connector-values" % flinkCdcVersion % Test
)
61 changes: 59 additions & 2 deletions src/test/scala/PhakerTest.scala
Original file line number Diff line number Diff line change
@@ -1,23 +1,80 @@
package io.github.yuxiqian.phaker

import factory.PhakerDataFactory
import source.PhakerSourceFunction

import org.apache.flink.cdc.common.event.TableId
import org.apache.flink.cdc.composer.definition.{SinkDef, SourceDef}
import org.apache.flink.cdc.composer.flink.FlinkPipelineComposer
import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory
import org.apache.flink.cdc.connectors.values.sink.{ValuesDataSink, ValuesDataSinkOptions}
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
import org.scalatest.funsuite.AnyFunSuite

class PhakerTest extends AnyFunSuite {

test("Phaker test example") {
import org.apache.flink.cdc.common.configuration.Configuration
import org.apache.flink.cdc.common.pipeline.PipelineOptions
import org.apache.flink.cdc.composer.definition.PipelineDef

import java.util.Collections

test("Phaker source test") {
val source = new PhakerSourceFunction(
TableId.tableId("default_namespace", "default_schema", "default_table"),
true,
17,
1,
17,
1000
)
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.addSource(source).print().setParallelism(1)
env.execute("Let's Test Phaker Source...")
}

test("Phaker to Values test") {
import source.PhakerDataSourceOptions

import org.apache.flink.cdc.composer.definition.{RouteDef, TransformDef}

val composer = FlinkPipelineComposer.ofMiniCluster

// Setup value source
val sourceConfig = new Configuration
sourceConfig
.set(PhakerDataSourceOptions.NAMESPACE_NAME, "default_namespace")
.set(PhakerDataSourceOptions.SCHEMA_NAME, "default_schema")
.set(PhakerDataSourceOptions.TABLE_NAME, "default_table")
.set[java.lang.Integer](PhakerDataSourceOptions.BATCH_COUNT, 1)
.set[java.lang.Integer](PhakerDataSourceOptions.MAX_COLUMN_COUNT, 50)
.set[java.lang.Integer](PhakerDataSourceOptions.SLEEP_TIME, 1000)

val sourceDef =
new SourceDef(PhakerDataFactory.IDENTIFIER, "Value Source", sourceConfig)

// Setup value sink
val sinkConfig = new Configuration
sinkConfig.set(
ValuesDataSinkOptions.SINK_API,
ValuesDataSink.SinkApi.SINK_V2
)
val sinkDef =
new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig)

// Setup pipeline
val pipelineConfig = new Configuration
pipelineConfig
.set[java.lang.Integer](PipelineOptions.PIPELINE_PARALLELISM, 1)
val pipelineDef = new PipelineDef(
sourceDef,
sinkDef,
Collections.emptyList[RouteDef],
Collections.emptyList[TransformDef],
pipelineConfig
)

// Execute the pipeline
val execution = composer.compose(pipelineDef)
execution.execute
}
}

0 comments on commit b72249c

Please sign in to comment.