docker-compose.yml

# OpenCue Docker Compose Configuration
#
# By default, only the core services are enabled (db, flyway, cuebot, rqd (Rust)).
# Uncomment the optional sections to enable additional functionality:
#
# Profiles:
#   - default (cuebot, db, flyway, rqd)
#   - cueweb (rest-gateway, cueweb)
#   - monitoring (db-exporter, prometheus, grafana, loki)
#   - monitoring-full (zookeeper, kafka, kafka-ui, elasticsearch, kibana, monitoring-indexer)
# Optional Components:
#   - Web UI (rest-gateway, cueweb): Web-based interface for monitoring and managing OpenCue
#   - Monitoring (prometheus, grafana, loki, db-exporter): Metrics and logging
#   - Event Streaming (kafka, zookeeper, elasticsearch, kibana, monitoring-indexer): Event streaming and historical data
#
# =============================================================================
# QUICK START
# =============================================================================
#
# 1. Prerequisites:
#    mkdir -p /tmp/rqd/logs /tmp/rqd/shots
#    docker compose --profile all down -v  # Clean up existing containers (optional)
#
# 2. Validate configuration:
#    docker compose --profile all config
#
# 3. Start services:
#
#    Core only (default):
#    docker compose up -d
#
#    Specific profile:
#    docker compose --profile <profile> up -d
#    # Full stack:
#    docker compose --profile all up -d
#
# 4. Check status:
#    docker compose ps
#    docker compose logs -f  # Watch logs (Ctrl+C to exit)
#
# 5. Troubleshoot:
#    docker compose logs <service-name>      # Check specific service logs
#    docker compose restart <service-name>   # Restart a specific service
#
# 6. CueWeb "ECONNREFUSED" or "/api/job/getjobs" errors:
#    This happens when CueWeb has http://localhost:8448 baked in instead of
#    http://rest-gateway:8448. Next.js NEXT_PUBLIC_* vars are embedded at build time.
#    Fix: Rebuild the image with correct build args:
#    docker compose build --no-cache cueweb && docker compose up -d cueweb
#
# 7. Enable Kafka monitoring for Cuebot:
#    In the cuebot service section:
#    a) Uncomment port 8080:
#       - "8080:8080"
#    b) Comment out the simple command and uncomment the Kafka command:
#       # command: --datasource.cue-data-source.jdbc-url=...
#       command:
#         - java
#         - -jar
#         - /opt/opencue/cuebot-latest.jar
#         - --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot
#         - --datasource.cue-data-source.username=cuebot
#         - --datasource.cue-data-source.password=cuebot_password
#         - --monitoring.kafka.enabled=true
#         - --monitoring.kafka.bootstrap.servers=kafka:29092
#         - --metrics.prometheus.collector=true
#    c) Restart cuebot: docker compose up -d cuebot
#
# =============================================================================
# ACCESS ENDPOINTS (when enabled)
# =============================================================================
#
#   Service          URL                              Credentials
#   -------          ---                              -----------
#   CueWeb UI        http://localhost:3000            -
#   REST Gateway     http://localhost:8448            -
#   Cuebot gRPC      localhost:8443                   -
#   PostgreSQL       localhost:5432                   cuebot / cuebot_password
#   Grafana          http://localhost:3001            admin / admin
#   Prometheus       http://localhost:9090            -
#   Loki             http://localhost:3100            -
#   Kafka UI         http://localhost:8090            -
#   Kibana           http://localhost:5601            -
#   Elasticsearch    http://localhost:9200            -

services:
  # =============================================================================
  # CORE SERVICES (enabled by default)
  # =============================================================================

  # PostgreSQL Database
  # Stores all OpenCue data including jobs, frames, hosts, shows, etc.
  db:
    image: postgres:15.1
    environment:
      - POSTGRES_USER=cuebot
      - POSTGRES_PASSWORD=cuebot_password
      - POSTGRES_DB=cuebot
    ports:
      - "5432:5432"
    volumes:
      - db-data:/var/lib/postgresql/data
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U cuebot -d cuebot"]
      interval: 10s
      timeout: 5s
      retries: 5
    # Uncomment for Loki logging (requires loki service)
    # logging:
    #   driver: loki
    #   options:
    #     loki-url: http://localhost:3100/loki/api/v1/push
    #     loki-external-labels: job=db

  # Flyway Database Migration
  # Applies database schema migrations to keep the database up to date
  flyway:
    build:
      context: ./
      dockerfile: ./sandbox/flyway.Dockerfile
    depends_on:
      db:
        condition: service_healthy
    environment:
      - PGUSER=cuebot
      - PGPASSWORD=cuebot_password
      - PGDATABASE=cuebot
      - PGHOST=db
      - PGPORT=5432
    command: /opt/scripts/migrate.sh

  # Cuebot Server
  # The main OpenCue server that manages jobs, frames, and render hosts
  cuebot:
    # Use build to compile from source with latest code (recommended for development)
    build:
      context: ./
      dockerfile: ./cuebot/Dockerfile
    # Use image for faster startup with pre-built image (uncomment to use)
    # image: opencue/cuebot
    ports:
      - "8443:8443"
      # Uncomment for metrics endpoint (used with Prometheus monitoring)
      # - "8080:8080"
    depends_on:
      db:
        condition: service_healthy
      flyway:
        condition: service_completed_successfully
      # Uncomment if using Kafka monitoring (also uncomment kafka service below)
      # kafka:
      #   condition: service_healthy
      #   required: false
    restart: always
    environment:
      - CUE_FRAME_LOG_DIR=/tmp/rqd/logs
    # Default command (works without Kafka)
    command: --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot --datasource.cue-data-source.username=cuebot --datasource.cue-data-source.password=cuebot_password
    # Alternative: Enable Kafka monitoring (requires kafka service running)
    # Note: The datasource parameters below configure the database connection and can be
    # modified to point to an external database. Only the monitoring.kafka.* and metrics.*
    # parameters are specific to Kafka event streaming.
    # command:
    #   - java
    #   - -jar
    #   - /opt/opencue/cuebot-latest.jar
    #   - --datasource.cue-data-source.jdbc-url=jdbc:postgresql://db/cuebot
    #   - --datasource.cue-data-source.username=cuebot
    #   - --datasource.cue-data-source.password=cuebot_password
    #   - --monitoring.kafka.enabled=true
    #   - --monitoring.kafka.bootstrap.servers=kafka:29092
    #   - --metrics.prometheus.collector=true
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8080/ || [ $? -eq 22 ]"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s
    # Uncomment for Loki logging (requires loki service)
    # logging:
    #   driver: loki
    #   options:
    #     loki-url: http://localhost:3100/loki/api/v1/push
    #     loki-external-labels: job=cuebot

  # RQD Render Queue Daemon
  # Runs on render hosts and executes frames assigned by Cuebot
  # Note: In a production environment, this should be disabled from the stack and only run on render hosts
  rqd:
    build:
      context: ./
      dockerfile: ./rust/Dockerfile.rqd
    environment:
      - OPENRQD__GRPC__CUEBOT_ENDPOINTS=cuebot:8443
      - OPENRQD__MACHINE__USE_IP_AS_HOSTNAME=true
    depends_on:
      cuebot:
        condition: service_healthy
    ports:
      - "8444:8444"
    volumes:
      - /tmp/rqd/logs:/tmp/rqd/logs
      - /tmp/rqd/shots:/tmp/rqd/shots
      # Mount session and temp dirs so rqd can access outline.yaml and user_dir
      # created by pycuerun on the host
      - ${HOME}/.opencue/sessions:${HOME}/.opencue/sessions
      - /tmp/opencue:/tmp/opencue
      - /var/folders:/var/folders:ro
    restart: unless-stopped
    # Uncomment for Loki logging (requires loki service)
    # logging:
    #   driver: loki
    #   options:
    #     loki-url: http://localhost:3100/loki/api/v1/push
    #     loki-external-labels: job=rqd

  # =============================================================================
  # WEB UI SERVICES (cueweb)
  # Use this profile to enable the web-based interface
  # =============================================================================

  # REST Gateway
  # Provides HTTP/REST API access to OpenCue (used by CueWeb)
  rest-gateway:
    profiles:
      - cueweb
      - all
    image: opencue/rest-gateway:latest
    build:
      context: ./
      dockerfile: rest_gateway/Dockerfile
    container_name: opencue-rest-gateway
    ports:
      - "8448:8448"
    environment:
      - CUEBOT_ENDPOINT=cuebot:8443
      - JWT_SECRET=${JWT_SECRET:-opencue-dev-jwt-secret-change-in-production}
      - REST_PORT=8448
      - LOG_LEVEL=info
    depends_on:
      cuebot:
        condition: service_healthy
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "sh", "-c", "curl -sf http://localhost:8448/ || [ $? -eq 22 ]"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  # CueWeb UI
  # Web-based user interface for monitoring and managing OpenCue
  cueweb:
    profiles:
      - cueweb
      - all
    image: opencue/cueweb:latest
    build:
      context: ./cueweb
      dockerfile: Dockerfile
      args:
        # Build-time environment variables for Next.js
        - NEXT_PUBLIC_OPENCUE_ENDPOINT=http://rest-gateway:8448
        - NEXT_PUBLIC_URL=http://localhost:3000
        - NEXTAUTH_URL=http://localhost:3000
        - NEXTAUTH_SECRET=cueweb-nextauth-secret-change-in-production
        # Authentication disabled by default for sandbox/development
        # To enable authentication, uncomment and set providers:
        # - NEXT_PUBLIC_AUTH_PROVIDER=github,okta,google,ldap
        - NEXT_PUBLIC_AUTH_PROVIDER=
        # Disable Sentry for sandbox
        - SENTRY_ENVIRONMENT=development
    container_name: opencue-cueweb
    ports:
      - "3000:3000"
    volumes:
      # Mount RQD logs directory so CueWeb can display frame logs
      - /tmp/rqd/logs:/tmp/rqd/logs:ro
    environment:
      # Runtime environment variables
      - NEXT_PUBLIC_OPENCUE_ENDPOINT=http://rest-gateway:8448
      - NEXT_PUBLIC_URL=http://localhost:3000
      - NEXT_JWT_SECRET=${JWT_SECRET:-opencue-dev-jwt-secret-change-in-production}
      - NEXTAUTH_URL=http://localhost:3000
      - NEXTAUTH_SECRET=cueweb-nextauth-secret-change-in-production
      - SENTRY_ENVIRONMENT=development
      # Authentication disabled - empty value skips auth providers
      - NEXT_PUBLIC_AUTH_PROVIDER=
    depends_on:
      rest-gateway:
        condition: service_healthy
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

  # =============================================================================
  # MONITORING SERVICES (monitoring)
  # Use the profile for metrics, dashboards, and log aggregation
  # =============================================================================

  # PostgreSQL Exporter for Prometheus
  db-exporter:
    profiles:
      - monitoring
      - monitoring-full
      - all
    image: prometheuscommunity/postgres-exporter:latest
    container_name: opencue-db-exporter
    environment:
      - DATA_SOURCE_URI=db:5432/postgres?sslmode=disable
      - DATA_SOURCE_USER=cuebot
      - DATA_SOURCE_PASS=cuebot_password
      - PG_EXPORTER_AUTO_DISCOVER_DATABASES=true
    ports:
      - "9187:9187"
    depends_on:
      - db
    restart: unless-stopped

  # Prometheus for metrics collection
  prometheus:
    profiles:
      - monitoring
      - monitoring-full
      - all
    image: prom/prometheus:v3.9.1
    container_name: opencue-prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./sandbox/config/prometheus-monitoring.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.enable-lifecycle'
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
      interval: 10s
      timeout: 5s
      retries: 3
    restart: unless-stopped

  # Grafana for dashboards
  # Port 3001 to avoid conflict with CueWeb (3000)
  grafana:
    profiles:
      - monitoring
      - monitoring-full
      - all
    image: grafana/grafana:12.3.5
    container_name: opencue-grafana
    depends_on:
      - prometheus
    ports:
      - "3001:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - ./sandbox/config/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./sandbox/config/grafana/dashboards:/etc/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana

  # Loki for log aggregation
  # Enable loki logging driver on other services to use
  loki:
    profiles:
      - monitoring
      - monitoring-full
      - all
    image: grafana/loki:3.6.0
    container_name: opencue-loki
    volumes:
      - ./sandbox/config/loki:/etc/config
    entrypoint:
      - /usr/bin/loki
      - -config.file=/etc/config/loki.yaml
    ports:
      - "3100:3100"
    restart: unless-stopped

  # =============================================================================
  # EVENT STREAMING SERVICES (monitoring-full)
  # Use this profile for Kafka event streaming and Elasticsearch historical data
  # Note: This requires significant resources and is intended for production-like setups
  # =============================================================================

  # Zookeeper for Kafka
  zookeeper:
    profiles:
      - monitoring-full
      - all
    image: confluentinc/cp-zookeeper:7.7.7
    hostname: zookeeper
    container_name: opencue-zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    healthcheck:
      test: ["CMD", "nc", "-z", "localhost", "2181"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  # Kafka broker for event streaming
  kafka:
    profiles:
      - monitoring-full
      - all
    image: confluentinc/cp-kafka:7.7.7
    hostname: kafka
    container_name: opencue-kafka
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    healthcheck:
      test: ["CMD", "kafka-topics", "--bootstrap-server", "kafka:29092", "--list"]
      interval: 10s
      timeout: 10s
      retries: 5

  # Kafka UI for debugging
  kafka-ui:
    profiles:
      - monitoring-full
      - all
    image: provectuslabs/kafka-ui:latest
    container_name: opencue-kafka-ui
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8090:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: opencue
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
    restart: unless-stopped

  # Elasticsearch for historical data storage
  elasticsearch:
    profiles:
      - monitoring-full
      - all
    image: docker.elastic.co/elasticsearch/elasticsearch:9.3.0
    container_name: opencue-elasticsearch
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
      - "9300:9300"
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    healthcheck:
      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"]
      interval: 10s
      timeout: 10s
      retries: 10

  # Kibana for Elasticsearch visualization
  kibana:
    profiles:
      - monitoring-full
      - all
    image: docker.elastic.co/kibana/kibana:9.3.0
    container_name: opencue-kibana
    depends_on:
      elasticsearch:
        condition: service_healthy
    ports:
      - "5601:5601"
    environment:
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    restart: unless-stopped

  # Monitoring indexer - indexes OpenCue events from Kafka to Elasticsearch
  # Note: Requires building from source (rust)
  monitoring-indexer:
    profiles:
      - monitoring-full
      - all
    build:
      context: ./rust
      dockerfile: Dockerfile.monitoring-indexer
    container_name: opencue-monitoring-indexer
    depends_on:
      kafka:
        condition: service_healthy
      elasticsearch:
        condition: service_healthy
    environment:
      - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
      - KAFKA_GROUP_ID=opencue-monitoring-indexer
      - ELASTICSEARCH_URL=http://elasticsearch:9200
      - ELASTICSEARCH_INDEX_PREFIX=opencue
      - LOG_LEVEL=info
    restart: unless-stopped

# =============================================================================
# VOLUMES
# =============================================================================

volumes:
  prometheus-data:
  grafana-data:
  elasticsearch-data:
  db-data: