From fab96ad05609a2cf1babe21e6cfda42cdef42227 Mon Sep 17 00:00:00 2001 From: Jiri Pechanec Date: Thu, 14 Mar 2024 14:30:20 +0100 Subject: [PATCH] DBZ-2002 Tutorial for Db2 iSeries --- tutorial/README.md | 40 +++++++++++- tutorial/debezium-ibmi-init/Dockerfile | 17 +++++ tutorial/debezium-ibmi-init/initdb.groovy | 28 +++++++++ tutorial/debezium-ibmi-init/inventory.sql | 77 +++++++++++++++++++++++ tutorial/docker-compose-ibmi.yaml | 33 ++++++++++ tutorial/register-ibmi.json | 17 +++++ 6 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tutorial/debezium-ibmi-init/Dockerfile create mode 100644 tutorial/debezium-ibmi-init/initdb.groovy create mode 100644 tutorial/debezium-ibmi-init/inventory.sql create mode 100644 tutorial/docker-compose-ibmi.yaml create mode 100644 tutorial/register-ibmi.json diff --git a/tutorial/README.md b/tutorial/README.md index 764f5b68..fa4a5b92 100644 --- a/tutorial/README.md +++ b/tutorial/README.md @@ -16,6 +16,7 @@ This demo automatically deploys the topology of services as defined in the [Debe * [Using Oracle](#using-oracle) * [Using SQL Server](#using-sql-server) * [Using Db2](#using-db2) + * [Using Db2 for iSeries](#using-db2-for-iseries) * [Using Cassandra](#using-cassandra) * [Using Vitess](#using-vitess) * [Using TimescaleDB](#using-timescaledb) @@ -328,7 +329,7 @@ export DEBEZIUM_VERSION=2.1 docker-compose -f docker-compose-db2.yaml up --build -# Start DB2 connector +# Start Db2 connector curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-db2.json # Consume messages from a Debezium topic @@ -338,7 +339,7 @@ docker-compose -f docker-compose-db2.yaml exec kafka /kafka/bin/kafka-console-co --property print.key=true \ --topic db2server.DB2INST1.CUSTOMERS -# Modify records in the database via DB2 client +# Modify records in the database via Db2 client docker-compose -f docker-compose-db2.yaml exec db2server bash -c 'su - db2inst1' db2 connect to TESTDB db2 "INSERT INTO DB2INST1.CUSTOMERS(first_name, last_name, email) VALUES ('John', 'Doe', 'john.doe@example.com');" @@ -346,6 +347,41 @@ db2 "INSERT INTO DB2INST1.CUSTOMERS(first_name, last_name, email) VALUES ('John' docker-compose -f docker-compose-db2.yaml down ``` +## Using Db2 for iSeries + +```shell +# Start the topology as defined in https://debezium.io/documentation/reference/stable/tutorial.html +export DEBEZIUM_VERSION=2.6 + +docker-compose -f docker-compose-ibmi.yaml up --build + +# Initialize datafiles and insert some test data +docker-compose -f docker-compose-ibmi.yaml exec connect bash -c 'groovy/bin/groovy initdb.groovy ' + +# Log into the database console and enable journal for the test data +# These and the following commands expects that the captured data are stored in DBZDATA library +STRJRNPF FILE(DBZDATA/PRODUCTS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH) +STRJRNPF FILE(DBZDATA/PRODUCTS_ON_HAND) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH) +STRJRNPF FILE(DBZDATA/ORDERS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH) +STRJRNPF FILE(DBZDATA/CUSTOMERS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH) + +# Start Db2 connector, it is necessary to replace placholders in register-ibmi.json file with real values +curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-ibmi.json + +# Consume messages from a Debezium topic +docker-compose -f docker-compose-ibmi.yaml exec kafka /kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server kafka:9092 \ + --from-beginning \ + --property print.key=true \ + --topic dbserver1.DBZDATA.CUSTOMERS + +# Modify records in the database via the script +docker-compose -f docker-compose-ibmi.yaml exec connect bash -c "groovy/bin/groovy initdb.groovy \"INSERT INTO DBZDATA.CUSTOMERS(first_name, last_name, email) VALUES ('John', 'Doe', 'john.doe@example.com')\"" + +# Shut down the cluster +docker-compose -f docker-compose-ibmi.yaml down +``` + ## Using Cassandra ```shell diff --git a/tutorial/debezium-ibmi-init/Dockerfile b/tutorial/debezium-ibmi-init/Dockerfile new file mode 100644 index 00000000..52639feb --- /dev/null +++ b/tutorial/debezium-ibmi-init/Dockerfile @@ -0,0 +1,17 @@ +ARG DEBEZIUM_VERSION +FROM quay.io/debezium/connect:nightly + +ENV GROOVY_VERSION=4.0.20 + +USER root +RUN microdnf -y install which curl java-17-openjdk-devel && microdnf clean all + +USER kafka + +# Deploy database init script +RUN curl https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/apache-groovy-binary-$GROOVY_VERSION.zip > groovy.zip &&\ + unzip groovy.zip &&\ + rm groovy.zip &&\ + mv groovy-$GROOVY_VERSION groovy + +COPY initdb.groovy inventory.sql . diff --git a/tutorial/debezium-ibmi-init/initdb.groovy b/tutorial/debezium-ibmi-init/initdb.groovy new file mode 100644 index 00000000..25bd4ad7 --- /dev/null +++ b/tutorial/debezium-ibmi-init/initdb.groovy @@ -0,0 +1,28 @@ +@Grab('net.sf.jt400:jt400:20.0.7') + +import java.sql.DriverManager + +def driver = new com.ibm.as400.access.AS400JDBCDriver() + +if (args.length < 4 || args.length > 5) { + println 'Usage: initdb.groovy []' + return -1 +} + +def host = args[0] +def dataLibrary = args[1] +def username = args[2] +def password = args[3] + +def sql = args.length > 4 ? [args[4]] : ('inventory.sql' as File).text.replace('##LIBRARY##', dataLibrary).trim().split(';') + +try ( + def connection = driver.connect("jdbc:as400://$host/", new Properties(['user': username, 'password': password])) + def statement = connection.createStatement() + ) { + + sql.each { + statement.executeUpdate(it) + } +} + diff --git a/tutorial/debezium-ibmi-init/inventory.sql b/tutorial/debezium-ibmi-init/inventory.sql new file mode 100644 index 00000000..2578fc64 --- /dev/null +++ b/tutorial/debezium-ibmi-init/inventory.sql @@ -0,0 +1,77 @@ + +-- Create and populate our products using a single insert with many rows +CREATE TABLE ##LIBRARY##.PRODUCTS ( + id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY + (START WITH 101, INCREMENT BY 1) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT +); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('scooter','Small 2-wheel scooter',3.14); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('car battery','12V car battery',8.1); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('12-pack drill bits','12-pack of drill bits with sizes ranging from #40 to #3',0.8); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('hammer','12oz carpenter''s hammer',0.75); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('hammer','14oz carpenter''s hammer',0.875); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('hammer','16oz carpenter''s hammer',1.0); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('rocks','box of assorted rocks',5.3); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('jacket','water resistent black wind breaker',0.1); +INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight) + VALUES ('spare tire','24 inch spare tire',22.2); + +CREATE TABLE ##LIBRARY##.PRODUCTS_ON_HAND ( + product_id INTEGER NOT NULL PRIMARY KEY, + quantity INTEGER NOT NULL, + FOREIGN KEY (product_id) REFERENCES ##LIBRARY##.PRODUCTS(id) +); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (101,3); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (102,8); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (103,18); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (104,4); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (105,5); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (106,0); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (107,44); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (108,2); +INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (109,5); + +CREATE TABLE ##LIBRARY##.CUSTOMERS ( + id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY + (START WITH 1001, INCREMENT BY 1) PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE +); +INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email) + VALUES ('Sally','Thomas','sally.thomas@acme.com'); +INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email) + VALUES ('George','Bailey','gbailey@foobar.com'); +INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email) + VALUES ('Edward','Walker','ed@walker.com'); +INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email) + VALUES ('Anne','Kretchmar','annek@noanswer.org'); + +CREATE TABLE ##LIBRARY##.ORDERS ( + id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY + (START WITH 10001, INCREMENT BY 1) PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY (purchaser) REFERENCES ##LIBRARY##.CUSTOMERS(id), + FOREIGN KEY (product_id) REFERENCES ##LIBRARY##.PRODUCTS(id) +); +INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id) + VALUES ('2016-01-16', 1001, 1, 102); +INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id) + VALUES ('2016-01-17', 1002, 2, 105); +INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id) + VALUES ('2016-02-19', 1002, 2, 106); +INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id) + VALUES ('2016-02-21', 1003, 1, 107); diff --git a/tutorial/docker-compose-ibmi.yaml b/tutorial/docker-compose-ibmi.yaml new file mode 100644 index 00000000..59db8e91 --- /dev/null +++ b/tutorial/docker-compose-ibmi.yaml @@ -0,0 +1,33 @@ +version: '2' +services: + zookeeper: + image: quay.io/debezium/zookeeper:${DEBEZIUM_VERSION} + ports: + - 2181:2181 + - 2888:2888 + - 3888:3888 + kafka: + image: quay.io/debezium/kafka:${DEBEZIUM_VERSION} + ports: + - 9092:9092 + links: + - zookeeper + environment: + - ZOOKEEPER_CONNECT=zookeeper:2181 + connect: + image: debezium/connect-ibmi:${DEBEZIUM_VERSION} + build: + context: ./debezium-ibmi-init + args: + DEBEZIUM_VERSION: ${DEBEZIUM_VERSION} + ports: + - 8083:8083 + - 5005:5005 + links: + - kafka + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=my_connect_configs + - OFFSET_STORAGE_TOPIC=my_connect_offsets + - STATUS_STORAGE_TOPIC=my_connect_statuses diff --git a/tutorial/register-ibmi.json b/tutorial/register-ibmi.json new file mode 100644 index 00000000..3857f24c --- /dev/null +++ b/tutorial/register-ibmi.json @@ -0,0 +1,17 @@ +{ + "name": "inventory-connector", + "config": { + "connector.class": "io.debezium.connector.db2as400.As400RpcConnector", + "tasks.max": "1", + "database.hostname": "", + "database.port": "", + "database.user": "", + "database.password": "", + "database.dbname": "", + "database.schema": "", + "database.secure": "false", + "secure": "false", + "topic.prefix": "dbserver1", + "table.include.list": ".CUSTOMERS" + } +}