Merge pull request #2 from SigNoz/docker-setup

fix consumer application issue and complete dockerized setup
SigNoz · Oct 1, 2024 · 68d3f1e · 68d3f1e
2 parents ac2a356 + a269669
commit 68d3f1e
Show file tree

Hide file tree

Showing 17 changed files with 351 additions and 160 deletions.
diff --git a/README.md b/README.md
@@ -4,9 +4,30 @@ Soon, we will be introducing deep correlation and insights with producer and con
 
 ![kafka-otel-signoz](assets/kafka-otel-signoz.png)
 
-**Note:**
-1) This guide is a reference for a production-grade kafka monitoring and instrumentation, this repo is intended to familiarize you with the involved complexities.
-2) All the tools used are open source and are licensed under Apache and MIT license.
+### Quick Start using docker
+#### start kafka
+```bash
+# from docker directory, cd docker
+docker compose -f kafka.yaml up
+```
+#### collector sending data to signoz
+```bash
+# from docker directory, cd docker
+docker compose -f collector.yaml up
+```
+
+#### metrics collector
+```bash
+# from collector directory, cd collector
+./otelcol-contrib --config collector-contrib-config.yaml
+```
+
+#### start producer and consumers
+```bash
+# from docker directory, cd docker
+docker compose -f producer-consumer.yaml up 
+```
+
 ---
 ### Steps to follow (ready-to-use guide)
 We will follow the following steps:
@@ -139,7 +160,7 @@ The last section contains some Troubleshooting tips!!
            -Dotel.instrumentation.kafka.producer-propagation.enabled=true \
            -Dotel.instrumentation.kafka.experimental-span-attributes=true \
            -Dotel.instrumentation.kafka.metric-reporter.enabled=true \
-           -jar ${PWD}/kafka-app-otel/kafka-consumer/target/kafka-consumer-1.0-SNAPSHOT-jar-with-dependencies.jar
+           -jar ${PWD}/kafka-app-otel/kafka-consumer/target/kafka-consumer.jar
    ```
 ---
 ### Step 4: SigNoz setup
@@ -166,10 +187,11 @@ e.g. if you're running the binary on your host machine, place it the root of the
 ```bash
 ./otelcol-contrib --config ${PWD}/collector/collector-contrib-config.yaml
 ```
----
-### Troubleshooting
-
 ---
 ### Contributing
 
 If you have some suggestions or want to improve the experience of the repo feel free to create an issue or open a pull request.
+
+**Note:**
+1) This guide is a reference for a production-grade kafka monitoring and instrumentation, this repo is intended to familiarize you with the involved complexities.
+2) All the tools used are open source and are licensed under Apache and MIT license.
diff --git a/collector/collector-contrib-config.yaml b/collector/collector-contrib-config.yaml
@@ -1,4 +1,12 @@
 receivers:
+  otlp:
+    protocols:
+      grpc:
+        # OTeL receiver endpoint (grpc)
+        endpoint: 127.0.0.1:4317
+      http:
+        # OTeL receiver endpoint (http)
+        endpoint: 127.0.0.1:4318
   kafkametrics:
     brokers:
       - localhost:9092
@@ -37,7 +45,7 @@ receivers:
 
 exporters:
   otlp:
-    endpoint: localhost:4317
+    endpoint: localhost:51375
     tls:
       insecure: true
   debug:
@@ -46,5 +54,11 @@ exporters:
 service:
   pipelines:
     metrics:
-      receivers: [kafkametrics, jmx/1, jmx/2, jmx/3]
+      receivers: [kafkametrics, jmx/1, jmx/2, jmx/3, otlp]
+      exporters: [otlp]
+    traces:
+      receivers: [otlp]
+      exporters: [otlp, debug]
+    logs:
+      receivers: [otlp]
       exporters: [otlp]
diff --git a/docker/consumer/Dockerfile b/docker/consumer/Dockerfile
@@ -1,5 +1,5 @@
 # Use an OpenJDK base image
-FROM openjdk:22-jdk-slim
+FROM openjdk:22-slim-bullseye
 
 # Create a directory for the application
 RUN mkdir -p /opt

diff --git a/docker/consumer/kafka-consumer.jar b/docker/consumer/kafka-consumer.jar
diff --git a/docker/producer-consumer.yaml b/docker/producer-consumer.yaml
@@ -11,9 +11,9 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
   kafka-producer2:
     image: shivanshu1333/kafka-producer:latest
@@ -27,9 +27,9 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
   kafka-producer3:
     image: shivanshu1333/kafka-producer:latest
@@ -42,9 +42,9 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
   kafka-producer4:
     image: shivanshu1333/kafka-producer:latest
@@ -57,9 +57,9 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
   kafka-producer5:
     image: shivanshu1333/kafka-producer:latest
@@ -72,9 +72,9 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
   kafka-producer6:
     image: shivanshu1333/kafka-producer:latest
@@ -87,96 +87,96 @@ services:
       OTEL_TRACES_EXPORTER: "otlp"
       OTEL_METRICS_EXPORTER: "otlp"
       OTEL_LOGS_EXPORTER: "otlp"
-      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 
-#  kafka-consumer1:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer1
-#    environment:
-#      BOOTSTRAP_SERVERS: "broker1:9092,broker2:19093,broker3:19094"
-#      CONSUMER_GROUP: "cg1"
-#      TOPIC: "topic1"
-#      OTEL_SERVICE_NAME: "consumer-svc"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "none"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
-#
-#  kafka-consumer2:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer2
-#    environment:
-#      BOOTSTRAP_SERVERS: "127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
-#      CONSUMER_GROUP: "cg1"
-#      TOPIC: "topic1"
-#      OTEL_SERVICE_NAME: "consumer-svc1"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "otlp"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
-##
-#  kafka-consumer3:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer3
-#    environment:
-#      BOOTSTRAP_SERVERS: "127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
-#      CONSUMER_GROUP: "cg2"
-#      TOPIC: "topic2"
-#      OTEL_SERVICE_NAME: "consumer-svc2"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "otlp"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
-##
-#  kafka-consumer4:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer4
-#    environment:
-#      BOOTSTRAP_SERVERS: "127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
-#      CONSUMER_GROUP: "cg2"
-#      TOPIC: "topic2"
-#      OTEL_SERVICE_NAME: "consumer-svc3"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "otlp"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+  kafka-consumer1:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer1
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg1"
+      TOPIC: "topic1"
+      OTEL_SERVICE_NAME: "consumer-svc"
+      OTEL_TRACES_EXPORTER: "none"
+      OTEL_METRICS_EXPORTER: "none"
+      OTEL_LOGS_EXPORTER: "none"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
+
+  kafka-consumer2:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer2
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg1"
+      TOPIC: "topic1"
+      OTEL_SERVICE_NAME: "consumer-svc1"
+      OTEL_TRACES_EXPORTER: "otlp"
+      OTEL_METRICS_EXPORTER: "otlp"
+      OTEL_LOGS_EXPORTER: "otlp"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 #
-#  kafka-consumer5:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer5
-#    environment:
-#      BOOTSTRAP_SERVERS: "127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
-#      CONSUMER_GROUP: "cg3"
-#      TOPIC: "topic3"
-#      OTEL_SERVICE_NAME: "consumer-svc3"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "otlp"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+  kafka-consumer3:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer3
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg2"
+      TOPIC: "topic2"
+      OTEL_SERVICE_NAME: "consumer-svc2"
+      OTEL_TRACES_EXPORTER: "otlp"
+      OTEL_METRICS_EXPORTER: "otlp"
+      OTEL_LOGS_EXPORTER: "otlp"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
 #
-#  kafka-consumer6:
-#    image: shivanshu1333/kafka-consumer:latest
-#    container_name: kafka-consumer6
-#    environment:
-#      BOOTSTRAP_SERVERS: "127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
-#      CONSUMER_GROUP: "cg3"
-#      TOPIC: "topic3"
-#      OTEL_SERVICE_NAME: "consumer-svc3"
-#      OTEL_TRACES_EXPORTER: "otlp"
-#      OTEL_METRICS_EXPORTER: "otlp"
-#      OTEL_LOGS_EXPORTER: "otlp"
-#      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://otel-collector:4318/v1/logs"
-#      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://otel-collector:4318/v1/metrics"
-#      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector:4318/v1/traces"
+  kafka-consumer4:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer4
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg2"
+      TOPIC: "topic2"
+      OTEL_SERVICE_NAME: "consumer-svc3"
+      OTEL_TRACES_EXPORTER: "otlp"
+      OTEL_METRICS_EXPORTER: "otlp"
+      OTEL_LOGS_EXPORTER: "otlp"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
+
+  kafka-consumer5:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer5
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg3"
+      TOPIC: "topic3"
+      OTEL_SERVICE_NAME: "consumer-svc3"
+      OTEL_TRACES_EXPORTER: "otlp"
+      OTEL_METRICS_EXPORTER: "otlp"
+      OTEL_LOGS_EXPORTER: "otlp"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
+
+  kafka-consumer6:
+    image: shivanshu1333/kafka-consumer:latest
+    container_name: consumer6
+    environment:
+      BOOTSTRAP_SERVERS: "broker1:19092,broker2:19093,broker3:19094"
+      CONSUMER_GROUP: "cg3"
+      TOPIC: "topic3"
+      OTEL_SERVICE_NAME: "consumer-svc3"
+      OTEL_TRACES_EXPORTER: "otlp"
+      OTEL_METRICS_EXPORTER: "otlp"
+      OTEL_LOGS_EXPORTER: "otlp"
+      OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "http://host.docker.internal:4318/v1/logs"
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://host.docker.internal:4318/v1/metrics"
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://host.docker.internal:4318/v1/traces"
diff --git a/docker/readme.md b/docker/readme.md
@@ -15,13 +15,13 @@ cd collector
 nohup ./otelcol-contrib --config collector-contrib-config.yaml > /dev/null 2>&1 & 
 ```
 
-## start producer
+## start producer and consumers
 ```bash
 cd docker
 nohup sudo docker compose -f producer-consumer.yaml up > /dev/null 2>&1 &
 ```
 
-## start consumer
+## (optional) start producer or consumer via script, this doesn't use the docker image
 ```bash
 cd scripts
 nohup ./consumer1.sh > /dev/null 2>&1 &