DBZ-7370 PubSub example for DS deployed via operator

operator/tutorial-pubsub/.gitignore

@@ -0,0 +1,2 @@
+service-account.json
+k8s/debezium/001_pubsub.yml

operator/tutorial-pubsub/README.md

@@ -0,0 +1,80 @@
+Deploying Debezium Server with Google Pub/Sub
+===
+This tutorial demonstrates how to stream changes from a PostgreSQL database into [Google Cloud Pub/Sub](https://cloud.google.com/pubsub/docs) with Debezium Server deployed in a Kubernetes cluster.
+
+Prerequisities
+---
+The following prerequisities are required 
+
+- Installed [Kind](https://kind.sigs.k8s.io/) to provision a local k8s cluster
+- Installed [kubectl](https://kubernetes.io/docs/reference/kubectl/) client to manage k8s cluster
+- Installed [gcloud](https://cloud.google.com/sdk/gcloud) CLI 
+- Google cloud service account [json key](https://console.cloud.google.com/apis/credentials)
+- Update `env.sh` (at minimum set the `PUBSUB_PROJECT_ID` and `PUBSUB_CREDENTIALS_FILE` variables)
+- Installed [jq]https://jqlang.github.io/jq/ too format messages pulled from Pub/Sub
+
+Creating Local Kubernetes Cluster
+---
+Simply run the following script
+
+```sh
+./create-environment.sh
+```
+
+Deploying Debezium Operator
+---
+To deploy Debezium operator create an OLM subscription 
+
+```sh
+kubectl create -f k8s/operator/001_subscription.yml 
+```
+
+Configure Pub/Sub Connection
+---
+
+Run the following script to initialize your Pub/Sub environment
+
+```sh
+./pubsub.sh
+```
+The script performs several steps
+
+- Generates k8s secret to `k8s/debezium/001_pubsub/yml` used by Debezium in the next step
+- Deletes Pub/Sub topic and subscription if any of them already exists
+- Creates the Pub/Sub topic and subscription anew
+
+Deploying Debezium Server with PostgreSQL Source and Pub/Sub Sink
+---
+With Debezium Operator up and running an instance of Debezium Server can be deployed into the kubernetes cluster by simply creating a `DebeziumServer` resource.
+
+```sh
+source env.sh
+kubectl create -f k8s/debezium/ -n $NAMESPACE
+```
+
+You can check that the deployed Debezium Server instance in running with the following command:
+
+```sh
+$ kubectl get deployments -n $NAMESPACE
+
+NAME                        READY   UP-TO-DATE   AVAILABLE 
+my-debezium                 1/1     1            1 
+```
+
+Verify Messages in Pub/Sub
+---
+To verify the messaged sent to Pub/Sub topic, pull your subscription by running the following script
+
+```shell
+./pull.sh 4
+```
+
+_Note: The number passed to the `pull.sh` script is the number of expected messages. In this case the connector emitted 4 messages for the initial data in the PostgreSQL database_
+
+Cleanup
+---
+To remove the Kubernetes environment used in this tutorial, execute the cleanup script:
+
+```sh
+./destroy-environment.sh
+```

operator/tutorial-pubsub/create-environment.sh

@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+
+source env.sh
+
+kind create cluster --name $CLUSTER
+kubectl cluster-info --context kind-$CLUSTER
+
+# Preload images
+docker pull quay.io/debezium/operator:$IMAGE_TAG
+docker pull quay.io/debezium/server:$IMAGE_TAG
+kind load docker-image quay.io/debezium/operator:$IMAGE_TAG --name $CLUSTER
+kind load docker-image quay.io/debezium/server:$IMAGE_TAG --name $CLUSTER
+
+# Create namespace
+echo ">>> Creating debezium namespace"
+kubectl create namespace $NAMESPACE
+kubectl config set-context --current --namespace $NAMESPACE
+
+# Deploy PostgreSQL database
+echo ">>> Deploying PostgreSQL"
+kubectl create -f k8s/database/001_postgresql.yml -n $NAMESPACE
+
+# Wait until envrionment is ready
+echo ">>> Waiting for Database to be ready"
+kubectl wait --for=condition=Available deployments/postgresql --timeout=$TIMEOUT -n $NAMESPACE
+
+echo ""
+echo "Kubernetes environment with PostreSQL database is ready"
+
+# Install OLM 
+echo ">>> Installing OLM to Kuberenetes cluster"
+./olm/install.sh v0.26.0

operator/tutorial-pubsub/destroy-environment.sh

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+source env.sh
+
+kind delete cluster --name $CLUSTER

operator/tutorial-pubsub/env.sh

@@ -0,0 +1,8 @@
+CLUSTER=debezium-demo
+NAMESPACE=demo
+TIMEOUT=300s
+IMAGE_TAG=2.5.0.Final
+PUBSUB_PROJECT_ID=change_me
+PUBSUB_CREDENTIALS_FILE=service-account.json
+PUBSUB_TOPIC=demo.inventory.customers
+PUBSUB_SUBSCRIPTION=demo

operator/tutorial-pubsub/k8s/database/001_postgresql.yml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgresql-credentials
+type: opaque
+data:
+  POSTGRES_DB: ZGViZXppdW0=
+  POSTGRES_USER: ZGViZXppdW0=
+  POSTGRES_PASSWORD: ZGViZXppdW0=
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: postgresql
+  labels:
+    app: postgresql
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgresql
+      deployment: postgresql
+  template:
+    metadata:
+      labels:
+        app: postgresql
+        deployment: postgresql
+    spec:
+      containers:
+        - resources: {}
+          name: postgresql
+          envFrom:
+            - secretRef:
+                name: postgresql-credentials
+          ports:
+            - containerPort: 5432
+              protocol: TCP
+          imagePullPolicy: IfNotPresent
+          livenessProbe:
+            initialDelaySeconds: 30
+            tcpSocket:
+              port: 5432
+            timeoutSeconds: 1
+          readinessProbe:
+            exec:
+              command:
+                - "/bin/sh"
+                - "-i"
+                - "-c"
+                - "PGPASSWORD=${POSTGRES_PASSWORD} /usr/bin/psql -w -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c 'SELECT 1'"
+            initialDelaySeconds: 5
+            timeoutSeconds: 1
+          terminationMessagePolicy: File
+          terminationMessagePath: /dev/termination-log
+          image: quay.io/debezium/example-postgres:latest
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+  strategy:
+    type: Recreate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgresql
+spec:
+  selector:
+    app: postgresql
+    deployment: postgresql
+  ports:
+    - name: db
+      port: 5432
+      targetPort: 5432

operator/tutorial-pubsub/k8s/debezium/002_debezium-server.yml

@@ -0,0 +1,42 @@
+apiVersion: debezium.io/v1alpha1
+kind: DebeziumServer
+metadata:
+  name: my-debezium
+spec:
+  quarkus:
+    config:
+      log.console.json: false
+      kubernetes-config.enabled: true
+      kubernetes-config.secrets: postgresql-credentials,pubsub-credentials
+  runtime:
+    volumes:
+      - name: pubsub
+        secret: 
+          secretName: pubsub-credentials
+          items:
+            - key: PUBSUB_CREDENTIALS
+              path: service-account.json
+    templates:
+      container:
+        env: 
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /debezium/external-configuration/pubsub/service-account.json
+  sink:
+    type: pubsub
+    config:
+      project.id: ${PUBSUB_PROJECT_ID}
+  source:
+    class: io.debezium.connector.postgresql.PostgresConnector
+    config:
+      tasks.max:
+      offset.storage.file.filename: /debezium/data/offsets.dat
+      database.history: io.debezium.relational.history.FileDatabaseHistory
+      database.hostname: postgresql
+      database.port: 5432
+      database.user: ${POSTGRES_USER}
+      database.password: ${POSTGRES_PASSWORD}
+      database.dbname: ${POSTGRES_DB}
+      topic.prefix: demo
+      table.include.list: inventory.customers
+      tombstones.on.delete: false
+

operator/tutorial-pubsub/k8s/operator/001_subscription.yml

@@ -0,0 +1,12 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: debezium-operator-subscription
+  namespace: operators
+spec:
+  installPlanApproval: Automatic
+  name: debezium-operator
+  source: operatorhubio-catalog
+  sourceNamespace: olm
+  channel: debezium-2.5.x
+  startingCSV: debezium-operator.v2.5.0

operator/tutorial-pubsub/olm/install.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+
+# This script is for installing OLM from a GitHub release
+
+set -e
+
+default_base_url=https://github.com/operator-framework/operator-lifecycle-manager/releases/download
+
+if [[ ${#@} -lt 1 || ${#@} -gt 2 ]]; then
+    echo "Usage: $0 version [base_url]"
+    echo "* version: the github release version"
+    echo "* base_url: the github base URL (Default: $default_base_url)"
+    exit 1
+fi
+
+if kubectl get deployment olm-operator -n openshift-operator-lifecycle-manager > /dev/null 2>&1; then
+    echo "OLM is already installed in a different configuration. This is common if you are not running a vanilla Kubernetes cluster. Exiting..."
+    exit 1
+fi
+
+release="$1"
+base_url="${2:-${default_base_url}}"
+url="${base_url}/${release}"
+namespace=olm
+
+if kubectl get deployment olm-operator -n ${namespace} > /dev/null 2>&1; then
+    echo "OLM is already installed in ${namespace} namespace. Exiting..."
+    exit 1
+fi
+
+kubectl create -f "${url}/crds.yaml"
+kubectl wait --for=condition=Established -f "${url}/crds.yaml"
+kubectl create -f "${url}/olm.yaml"
+
+# wait for deployments to be ready
+kubectl rollout status -w deployment/olm-operator --namespace="${namespace}"
+kubectl rollout status -w deployment/catalog-operator --namespace="${namespace}"
+
+retries=30
+until [[ $retries == 0 ]]; do
+    new_csv_phase=$(kubectl get csv -n "${namespace}" packageserver -o jsonpath='{.status.phase}' 2>/dev/null || echo "Waiting for CSV to appear")
+    if [[ $new_csv_phase != "$csv_phase" ]]; then
+        csv_phase=$new_csv_phase
+        echo "Package server phase: $csv_phase"
+    fi
+    if [[ "$new_csv_phase" == "Succeeded" ]]; then
+	break
+    fi
+    sleep 10
+    retries=$((retries - 1))
+done
+
+if [ $retries == 0 ]; then
+    echo "CSV \"packageserver\" failed to reach phase succeeded"
+    exit 1
+fi
+
+kubectl rollout status -w deployment/packageserver --namespace="${namespace}"

operator/tutorial-pubsub/pubsub.sh

@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+source env.sh
+
+PUBSUB_SECRET_MANIFEST="k8s/debezium/001_pubsub.yml"
+
+# Generate pubsub credentials secret
+echo ""
+echo "Generating $PUBSUB_SECRET_MANIFEST file"
+
+PUBSUB_CREDENTIALS_BASE64=$(cat $PUBSUB_CREDENTIALS_FILE | base64 -w0)
+
+cat << EOF > $PUBSUB_SECRET_MANIFEST
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pubsub-credentials
+type: opaque
+stringData:
+  PUBSUB_PROJECT_ID: $PUBSUB_PROJECT_ID
+data:
+  PUBSUB_CREDENTIALS: $PUBSUB_CREDENTIALS_BASE64
+EOF
+
+# Create new topic and subscription
+echo ""
+echo "(Re)creating pubsub topic and subscription"
+
+gcloud pubsub topics delete $PUBSUB_TOPIC
+gcloud pubsub subscriptions delete $PUBSUB_SUBSCRIPTION
+
+gcloud pubsub topics create $PUBSUB_TOPIC
+gcloud pubsub subscriptions create $PUBSUB_SUBSCRIPTION --topic $PUBSUB_TOPIC --enable-message-ordering

operator/tutorial-pubsub/pull.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+source env.sh
+
+gcloud pubsub subscriptions pull $PUBSUB_SUBSCRIPTION --auto-ack --format=json --limit=$1 \
+| jq  -r '.[].message.data | @base64d' \
+| jq .payload