Merge pull request #1 from SigNoz/vm-setup

Merge the docker changes to master

README.md

@@ -89,7 +89,7 @@ The last section contains some Troubleshooting tips!!
   ```
   ```bash
   # open second terminal and consume the messages
-  bin/kafka-console-consumer.sh --topic topic1 --from-beginning --bootstrap-server localhost:9092
+  bin/kafka-console-consumer1.sh --topic topic1 --from-beginning --bootstrap-server localhost:9092
   > (receive the messages)
   ```
 ---

collector/collector-contrib-config.yaml

@@ -1,58 +1,50 @@
 receivers:
-  otlp:
-    protocols:
-      grpc:
-        # OTeL receiver endpoint (grpc)
-        endpoint: 127.0.0.1:4317
-      http:
-        # OTeL receiver endpoint (http)
-        endpoint: 127.0.0.1:4318
   kafkametrics:
     brokers:
-      # Kafka brokers endpoints
-      - 127.0.0.1:9092
-      - 127.0.0.1:9093
+      - localhost:9092
+      - localhost:9093
+      - localhost:9094
     protocol_version: 2.0.0
     scrapers:
       - brokers
       - topics
       - consumers
-  jmx:
-    # configure the path where you installed opentelemetry-jmx-metrics receiver
+  jmx/1:
+    # configure the path where you installed opentelemetry-jmx-metrics jar
     jar_path: ${PWD}/opentelemetry-jmx-metrics.jar
-    endpoint: service:jmx:rmi:///jndi/rmi://127.0.0.1:2020/jmxrmi
+    endpoint: service:jmx:rmi:///jndi/rmi://localhost:9991/jmxrmi
     target_system: jvm,kafka,kafka-consumer,kafka-producer
     collection_interval: 10s
     log_level: info
     resource_attributes:
       broker.name: broker1
   jmx/2:
-    # configure the path where you installed opentelemetry-jmx-metrics receiver
     jar_path: ${PWD}/opentelemetry-jmx-metrics.jar
-    endpoint: service:jmx:rmi:///jndi/rmi://127.0.0.1:2021/jmxrmi
+    endpoint: service:jmx:rmi:///jndi/rmi://localhost:9992/jmxrmi
     target_system: jvm,kafka,kafka-consumer,kafka-producer
     collection_interval: 10s
     log_level: info
     resource_attributes:
       broker.name: broker2
+  jmx/3:
+    jar_path: ${PWD}/opentelemetry-jmx-metrics.jar
+    endpoint: service:jmx:rmi:///jndi/rmi://localhost:9993/jmxrmi
+    target_system: jvm,kafka,kafka-consumer,kafka-producer
+    collection_interval: 10s
+    log_level: info
+    resource_attributes:
+      broker.name: broker3
 
 exporters:
   otlp:
-    # configure the grpc endpoint for signoz otel collector
-    endpoint: "127.0.0.1:65118"
+    endpoint: localhost:4317
     tls:
       insecure: true
   debug:
     verbosity: detailed
 
 service:
   pipelines:
-    traces:
-      receivers: [otlp]
-      exporters: [otlp]
-    logs:
-      receivers: [otlp]
-      exporters: [otlp]
     metrics:
-      receivers: [otlp, kafkametrics]
-      exporters: [otlp, debug]
+      receivers: [kafkametrics, jmx/1, jmx/2, jmx/3]
+      exporters: [otlp]

docker/collector-config.yaml

@@ -0,0 +1,35 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  otlp/s:
+    endpoint: "10.128.0.35:4317"
+    tls:
+      insecure: true
+  otlp/t:
+    endpoint: "10.128.0.36:4317"
+    tls:
+      insecure: true
+  otlp/d:
+    endpoint: "10.128.0.34:4317"
+    tls:
+      insecure: true
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp/s, otlp/d, otlp/t]
+    logs:
+      receivers: [otlp]
+      exporters: [otlp/s, otlp/d, otlp/t]
+    metrics:
+      receivers: [otlp]
+      exporters: [otlp/s, otlp/d, otlp/t]

docker/collector.yaml

@@ -0,0 +1,9 @@
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector:latest
+    command: ["--config", "/etc/otelcol-contrib/config.yaml"]
+    volumes:
+      - ./collector-config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - 4317:4317
+      - 4318:4318

docker/consumer/Dockerfile

@@ -0,0 +1,25 @@
+# Use an OpenJDK base image
+FROM openjdk:22-jdk-slim
+
+# Create a directory for the application
+RUN mkdir -p /opt
+
+# Copy the JAR files into the Docker image
+COPY opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar
+COPY kafka-consumer.jar /opt/kafka-consumer.jar
+
+# Set environment variables
+ENV BOOTSTRAP_SERVERS="127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094"
+ENV CONSUMER_GROUP="cg1"
+ENV TOPIC="topic1"
+ENV OTEL_SERVICE_NAME="consumer-svc"
+ENV OTEL_TRACES_EXPORTER="otlp"
+ENV OTEL_METRICS_EXPORTER="otlp"
+ENV OTEL_LOGS_EXPORTER="otlp"
+
+# Define the entry point for the container
+ENTRYPOINT ["java", "-javaagent:/opt/opentelemetry-javaagent.jar", \
+            "-Dotel.instrumentation.kafka.producer-propagation.enabled=true", \
+            "-Dotel.instrumentation.kafka.experimental-span-attributes=true", \
+            "-Dotel.instrumentation.kafka.metric-reporter.enabled=true", \
+            "-jar", "/opt/kafka-consumer.jar"]

docker/kafka.yaml

@@ -0,0 +1,92 @@
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.5.0
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker1:
+    image: confluentinc/cp-server:7.5.0
+    hostname: broker1
+    container_name: broker1
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker1:19092,EXTERNAL://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19092,EXTERNAL://0.0.0.0:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9991"
+      JMX_PORT: 9991
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - "9092:9092"
+      - "9991:9991"
+
+  broker2:
+    image: confluentinc/cp-server:7.5.0
+    hostname: broker2
+    container_name: broker2
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker2:19093,EXTERNAL://localhost:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19093,EXTERNAL://0.0.0.0:9093
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9992"
+      JMX_PORT: 9992
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - "9093:9093"
+      - "9992:9992"
+
+  broker3:
+    image: confluentinc/cp-server:7.5.0
+    hostname: broker3
+    container_name: broker3
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker3:19094,EXTERNAL://localhost:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19094,EXTERNAL://0.0.0.0:9094
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9993"
+      JMX_PORT: 9993
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - "9094:9094"
+      - "9993:9993"
+
+  init-kafka:
+    image: confluentinc/cp-server:7.5.0
+    depends_on:
+      - broker1
+      - broker2
+      - broker3
+    entrypoint: ["/bin/sh", "-c"]
+    command: |
+      "
+      sleep 15
+      /usr/bin/kafka-topics --create --topic topic1 --partitions 3 --replication-factor 3 --bootstrap-server broker1:19092
+      /usr/bin/kafka-topics --create --topic topic2 --partitions 3 --replication-factor 3 --bootstrap-server broker1:19092
+      /usr/bin/kafka-topics --create --topic topic3 --partitions 3 --replication-factor 3 --bootstrap-server broker1:19092
+      kafka-topics --bootstrap-server broker1:19092 --list
+      "