feat(core, api, cli, ui, docs): integrate OpenTelemetry tracing in server (#46) (#47)

* feat(docker): add Jaeger service for OpenTelemetry tracing

* feat(api): integrate OpenTelemetry tracing

* feat(core): integrate OpenTelemetry tracing

* fix(docker): restore Jaeger service condition in docker-compose and remove unused server binary

* fix(api): initialize OpenTelemetry propagator in mock CoreClient to prevent nil pointer panic

* chore(api): update dependencies and integrate OpenTelemetry middleware for S3

* feat(core): enhance OpenTelemetry integration

* feat(core): add new OpenTelemetry instrumentation packages for enhanced monitoring

* fix(docker): update SeaweedFS image version to 3.96 in docker-compose

* feat(core): implement OpenTelemetry tracing for message processing and publishing in async_mq.py

* fix(api): use request context in project authentication middleware for improved database queries

* fix(docker): update RabbitMQ, SeaweedFS, AWS CLI, and Jaeger image versions in docker-compose

* fix(cli): enhance docker-compose with Jaeger service and update healthcheck syntax for improved readability

* feat(core): implement comprehensive OpenTelemetry configuration and error handling across services

* feat(core): add OpenTelemetry instrumentation for S3 client operations

* chore: update local_dev script

* fix(core): use core config for otel

* fix(core): remove unused

* feat(core): enhance OpenTelemetry instrumentation for LLM functions and streamline tracing setup in API

* fix(api): edit OpenTelemetry middleware

* feat(ui): add Jaeger API endpoints for availability check, services retrieval, and traces listing

* feat(docker): add Jaeger UI URLs to client and server docker-compose configurations

* feat(ui): add spacing to layout components and introduce IonLanguage icon for language switcher

* refactor(ui): skills temporarily hide dropdown menu for page type selection

* feat(docs): update dashboard images and add new traces viewer image

* feat(ui): add countdown timer and last refresh time display to traces page

---------

Co-authored-by: Gus <[email protected]>

Author: GenerQAQ

docs/images/dashboard/BI.png

@@ -14,7 +14,11 @@ services:
     volumes:
       - ${DATABASE_LOCATION:-./acontext_data/pg}:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${DATABASE_USER:-acontext} -d ${DATABASE_NAME:-acontext}"]
+      test:
+        [
+          "CMD-SHELL",
+          "pg_isready -U ${DATABASE_USER:-acontext} -d ${DATABASE_NAME:-acontext}",
+        ]
       interval: 30s
       timeout: 5s
       retries: 5
@@ -37,7 +41,7 @@ services:
 
   # --- RabbitMQ ---
   acontext-server-rabbitmq:
-    image: rabbitmq:3-management
+    image: rabbitmq:4-management
     container_name: acontext-server-rabbitmq
     restart: unless-stopped
     environment:
@@ -57,7 +61,7 @@ services:
 
   # --- SeaweedFS ---
   acontext-server-seaweedfs:
-    image: chrislusf/seaweedfs:latest
+    image: chrislusf/seaweedfs:4.00
     container_name: acontext-server-seaweedfs
     restart: unless-stopped
     user: root  # https://github.com/seaweedfs/seaweedfs/pull/7399
@@ -96,7 +100,7 @@ services:
 
   # init Bucket for SeaweedFS
   acontext-server-seaweedfs-setup:
-    image: amazon/aws-cli:latest
+    image: amazon/aws-cli:2.32.6
     container_name: acontext-server-seaweedfs-setup
     depends_on:
       acontext-server-seaweedfs:
@@ -113,6 +117,33 @@ services:
       "
     restart: "no"
 
+  # --- Jaeger (OpenTelemetry Collector) ---
+  acontext-server-jaeger:
+    image: jaegertracing/all-in-one:1.75.0
+    container_name: acontext-server-jaeger
+    restart: unless-stopped
+    user: root  # Run as root to ensure write permissions to mounted volume
+    environment:
+      COLLECTOR_OTLP_ENABLED: "true"
+      # Use Badger as storage backend (local file storage, suitable for development)
+      SPAN_STORAGE_TYPE: "badger"
+      BADGER_EPHEMERAL: "false"  # Disable ephemeral storage, enable persistence
+      BADGER_DIRECTORY: "/badger/data"
+      BADGER_SPAN_STORE_TTL: "168h"  # Data retention: 7 days
+    ports:
+      - "${JAEGER_UI_EXPORT_PORT:-16686}:16686"  # Jaeger UI
+      - "${JAEGER_OTLP_GRPC_EXPORT_PORT:-4317}:4317"  # OTLP gRPC receiver
+      - "${JAEGER_OTLP_HTTP_EXPORT_PORT:-4318}:4318"  # OTLP HTTP receiver
+      - "14250:14250"  # gRPC (model.proto)
+      - "14268:14268"  # HTTP (thrift)
+    volumes:
+      - ${JAEGER_STORAGE_LOCATION:-./acontext_data/jaeger}:/badger/data
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:16686"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+
   # acontext-server-core
   acontext-server-core:
     image: ${CORE_IMAGE:-ghcr.io/memodb-io/acontext-core:latest}
@@ -127,6 +158,7 @@ services:
       MQ_URL: amqp://${RABBITMQ_USER:-acontext}:${RABBITMQ_PASSWORD:-helloworld}@acontext-server-rabbitmq:5672/
       REDIS_URL: redis://:${REDIS_PASSWORD:-helloworld}@acontext-server-redis:6379
       S3_ENDPOINT: http://acontext-server-seaweedfs:9000
+      OTEL_EXPORTER_OTLP_ENDPOINT: acontext-server-jaeger:4317
     ports:
       - "${CORE_EXPORT_PORT:-8019}:8000"
     volumes:
@@ -142,6 +174,8 @@ services:
         condition: service_healthy
       acontext-server-seaweedfs-setup:
         condition: service_completed_successfully
+      acontext-server-jaeger:
+        condition: service_healthy
 
   # acontext-server-api
   acontext-server-api:
@@ -173,10 +207,16 @@ services:
       S3_SECRET_KEY: ${S3_SECRET_KEY:-helloworld}
       S3_BUCKET: ${S3_BUCKET:-acontext-assets}
       CORE_BASE_URL: http://acontext-server-core:8000
+      OTEL_EXPORTER_OTLP_ENDPOINT: acontext-server-jaeger:4317
+      APP_ENV: ${APP_ENV:-development}
     ports:
       - "${API_EXPORT_PORT:-8029}:8029"
     healthcheck:
-      test: ["CMD-SHELL", "wget -q -O- http://acontext-server-api:8029/health || exit 1"]
+      test:
+        [
+          "CMD-SHELL",
+          "wget -q -O- http://acontext-server-api:8029/health || exit 1",
+        ]
       interval: 30s
       timeout: 5s
       retries: 5
@@ -199,6 +239,8 @@ services:
       API_SERVER_URL: http://acontext-server-api:8029
       ROOT_API_BEARER_TOKEN: ${ROOT_API_BEARER_TOKEN:-your-root-api-bearer-token}
       DATABASE_URL: postgresql://${DATABASE_USER:-acontext}:${DATABASE_PASSWORD:-helloworld}@acontext-server-pg:5432/${DATABASE_NAME:-acontext}
+      JAEGER_UI_URL: http://acontext-server-jaeger:16686
+      NEXT_PUBLIC_JAEGER_UI_URL: http://acontext-server-jaeger:16686
     ports:
       - "${UI_EXPORT_PORT:-3000}:3000"
     depends_on:

docs/images/dashboard/artifact_viewer.png

@@ -24,9 +24,13 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/memodb-io/Acontext/internal/bootstrap"
 	"github.com/memodb-io/Acontext/internal/config"
+	"github.com/memodb-io/Acontext/internal/infra/cache"
+	dbpkg "github.com/memodb-io/Acontext/internal/infra/db"
 	"github.com/memodb-io/Acontext/internal/modules/handler"
 	"github.com/memodb-io/Acontext/internal/pkg/tokenizer"
 	"github.com/memodb-io/Acontext/internal/router"
+	"github.com/memodb-io/Acontext/internal/telemetry"
+	"github.com/redis/go-redis/v9"
 	"github.com/samber/do"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
@@ -39,12 +43,42 @@ func main() {
 	cfg := do.MustInvoke[*config.Config](inj)
 	log := do.MustInvoke[*zap.Logger](inj)
 	db := do.MustInvoke[*gorm.DB](inj)
+	rdb := do.MustInvoke[*redis.Client](inj)
 
 	// Initialize tokenizer (vocabulary is already embedded in the package)
 	if err := tokenizer.Init(log); err != nil {
 		log.Sugar().Fatalw("failed to initialize tokenizer", "err", err)
 	}
 
+	// Setup OpenTelemetry tracing (using configuration system)
+	tp, err := telemetry.SetupTracing(cfg)
+	if err != nil {
+		log.Sugar().Warnw("failed to setup tracing, continuing without tracing", "err", err)
+	} else if tp != nil {
+		log.Sugar().Info("OpenTelemetry tracing enabled", "endpoint", cfg.Telemetry.OtlpEndpoint)
+		defer func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			if err := telemetry.Shutdown(ctx); err != nil {
+				log.Sugar().Errorw("failed to shutdown tracer", "err", err)
+			}
+		}()
+
+		// Register GORM OpenTelemetry plugin after tracer provider is set
+		if err := dbpkg.RegisterOpenTelemetryPlugin(db); err != nil {
+			log.Sugar().Warnw("failed to register GORM OpenTelemetry plugin, continuing without database tracing", "err", err)
+		} else {
+			log.Sugar().Info("GORM OpenTelemetry plugin registered")
+		}
+
+		// Register Redis OpenTelemetry plugin after tracer provider is set
+		if err := cache.RegisterOpenTelemetryPlugin(rdb); err != nil {
+			log.Sugar().Warnw("failed to register Redis OpenTelemetry plugin, continuing without Redis tracing", "err", err)
+		} else {
+			log.Sugar().Info("Redis OpenTelemetry plugin registered")
+		}
+	}
+
 	// init gin
 	gin.SetMode(cfg.App.Env)
 

docs/images/dashboard/message_viewer.png

@@ -40,3 +40,8 @@ s3:
 
 core:
   baseURL: "${CORE_BASE_URL}"
+
+telemetry:
+  otlpEndpoint: "${OTEL_EXPORTER_OTLP_ENDPOINT}"
+  enabled: true
+  sampleRatio: 1.0  # Sampling ratio, 0.0-1.0, default 1.0 (100%)